{"level":"info","ts":"2026-06-05T09:28:03.907Z","logger":"setup","msg":"Loaded feature gates","featureGates":{"RayClusterStatusConditions":true,"RayJobDeletionPolicy":false}}
{"level":"info","ts":"2026-06-05T09:28:03.907Z","logger":"setup","msg":"Flag watchNamespace is not set. Watch custom resources in all namespaces."}
{"level":"info","ts":"2026-06-05T09:28:03.907Z","logger":"setup","msg":"Setup manager"}
{"level":"info","ts":"2026-06-05T09:28:04.206Z","logger":"setup","msg":"NetworkPolicy controller registered (annotation-based activation)"}
{"level":"info","ts":"2026-06-05T09:28:04.206Z","logger":"controller-runtime.builder","msg":"Registering a mutating webhook","GVK":"ray.io/v1, Kind=RayCluster","path":"/mutate-ray-io-v1-raycluster"}
{"level":"info","ts":"2026-06-05T09:28:04.206Z","logger":"controller-runtime.webhook","msg":"Registering webhook","path":"/mutate-ray-io-v1-raycluster"}
{"level":"info","ts":"2026-06-05T09:28:04.206Z","logger":"controller-runtime.builder","msg":"Registering a validating webhook","GVK":"ray.io/v1, Kind=RayCluster","path":"/validate-ray-io-v1-raycluster"}
{"level":"info","ts":"2026-06-05T09:28:04.206Z","logger":"controller-runtime.webhook","msg":"Registering webhook","path":"/validate-ray-io-v1-raycluster"}
{"level":"info","ts":"2026-06-05T09:28:04.206Z","logger":"setup","msg":"starting manager"}
{"level":"info","ts":"2026-06-05T09:28:04.206Z","logger":"controller-runtime.metrics","msg":"Starting metrics server"}
{"level":"info","ts":"2026-06-05T09:28:04.206Z","msg":"starting server","name":"health probe","addr":"[::]:8082"}
{"level":"info","ts":"2026-06-05T09:28:04.206Z","logger":"controller-runtime.webhook","msg":"Starting webhook server"}
{"level":"info","ts":"2026-06-05T09:28:04.206Z","logger":"controller-runtime.metrics","msg":"Serving metrics server","bindAddress":":8080","secure":false}
{"level":"info","ts":"2026-06-05T09:28:04.206Z","logger":"controller-runtime.certwatcher","msg":"Updated current TLS certificate","cert":"/tmp/k8s-webhook-server/serving-certs/tls.crt","key":"/tmp/k8s-webhook-server/serving-certs/tls.key"}
{"level":"info","ts":"2026-06-05T09:28:04.207Z","logger":"controller-runtime.webhook","msg":"Serving webhook server","host":"","port":9443}
{"level":"info","ts":"2026-06-05T09:28:04.207Z","logger":"controller-runtime.certwatcher","msg":"Starting certificate poll+watcher","cert":"/tmp/k8s-webhook-server/serving-certs/tls.crt","key":"/tmp/k8s-webhook-server/serving-certs/tls.key","interval":10}
{"level":"info","ts":"2026-06-05T09:28:04.507Z","msg":"attempting to acquire leader lease opendatahub/ray-operator-leader..."}
{"level":"info","ts":"2026-06-05T09:28:04.518Z","msg":"successfully acquired lease opendatahub/ray-operator-leader"}
{"level":"info","ts":"2026-06-05T09:28:04.518Z","logger":"controllers.RayCluster","msg":"Starting EventSource","source":"kind source: *v1.Pod"}
{"level":"info","ts":"2026-06-05T09:28:04.518Z","logger":"controllers.RayJob","msg":"Starting EventSource","source":"kind source: *v1.Job"}
{"level":"info","ts":"2026-06-05T09:28:04.518Z","logger":"controllers.RayJob","msg":"Starting EventSource","source":"kind source: *v1.RayCluster"}
{"level":"info","ts":"2026-06-05T09:28:04.518Z","logger":"controllers.RayJob","msg":"Starting EventSource","source":"kind source: *v1.RayJob"}
{"level":"info","ts":"2026-06-05T09:28:04.518Z","logger":"controllers.RayCluster","msg":"Starting EventSource","source":"kind source: *v1.RayCluster"}
{"level":"info","ts":"2026-06-05T09:28:04.518Z","logger":"controllers.RayCluster","msg":"Starting EventSource","source":"kind source: *v1.Service"}
{"level":"info","ts":"2026-06-05T09:28:04.518Z","logger":"controllers.RayService","msg":"Starting EventSource","source":"kind source: *v1.Service"}
{"level":"info","ts":"2026-06-05T09:28:04.518Z","logger":"controllers.RayJob","msg":"Starting EventSource","source":"kind source: *v1.Service"}
{"level":"info","ts":"2026-06-05T09:28:04.518Z","logger":"controllers.RayService","msg":"Starting EventSource","source":"kind source: *v1.RayService"}
{"level":"info","ts":"2026-06-05T09:28:04.519Z","logger":"controllers.RayService","msg":"Starting EventSource","source":"kind source: *v1.RayCluster"}
{"level":"info","ts":"2026-06-05T09:28:04.519Z","msg":"Starting EventSource","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","source":"kind source: *v1.RayCluster"}
{"level":"info","ts":"2026-06-05T09:28:04.519Z","msg":"Starting EventSource","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","source":"kind source: *v1.NetworkPolicy"}
{"level":"info","ts":"2026-06-05T09:28:04.519Z","msg":"Starting EventSource","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","source":"kind source: *v1.RayCluster"}
{"level":"info","ts":"2026-06-05T09:28:04.519Z","msg":"Starting EventSource","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","source":"kind source: *v1.RayCluster"}
{"level":"info","ts":"2026-06-05T09:28:04.519Z","msg":"Starting EventSource","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","source":"kind source: *v1.Route"}
{"level":"info","ts":"2026-06-05T09:28:04.519Z","msg":"Starting EventSource","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","source":"kind source: *v1.ServiceAccount"}
{"level":"info","ts":"2026-06-05T09:28:04.519Z","msg":"Starting EventSource","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","source":"kind source: *v1.Service"}
{"level":"info","ts":"2026-06-05T09:28:04.906Z","logger":"controllers.RayService","msg":"Starting Controller"}
{"level":"info","ts":"2026-06-05T09:28:04.906Z","logger":"controllers.RayService","msg":"Starting workers","worker count":1}
{"level":"info","ts":"2026-06-05T09:28:04.907Z","msg":"Starting Controller","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster"}
{"level":"info","ts":"2026-06-05T09:28:04.907Z","msg":"Starting workers","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","worker count":1}
{"level":"info","ts":"2026-06-05T09:28:04.907Z","msg":"Starting Controller","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster"}
{"level":"info","ts":"2026-06-05T09:28:04.907Z","msg":"Starting workers","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","worker count":1}
{"level":"info","ts":"2026-06-05T09:28:04.907Z","logger":"controllers.RayCluster","msg":"Starting Controller"}
{"level":"info","ts":"2026-06-05T09:28:04.907Z","logger":"controllers.RayCluster","msg":"Starting workers","worker count":1}
{"level":"info","ts":"2026-06-05T09:28:04.907Z","logger":"controllers.RayJob","msg":"Starting Controller"}
{"level":"info","ts":"2026-06-05T09:28:04.907Z","logger":"controllers.RayJob","msg":"Starting workers","worker count":1}
{"level":"info","ts":"2026-06-05T09:28:04.907Z","msg":"Starting Controller","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster"}
{"level":"info","ts":"2026-06-05T09:28:04.907Z","msg":"Starting workers","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","worker count":1}
{"level":"info","ts":"2026-06-05T09:30:41.552Z","logger":"raycluster-resource","msg":"default","name":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:30:41.552Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"raycluster-auth","namespace":"test-ns-69tjt"}
{"level":"info","ts":"2026-06-05T09:30:41.561Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"b0df6b25-cc10-48b9-a229-1fd7b0ae42c0","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:30:41.561Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"3e37b53a-96b1-45bc-8a51-e546cec2fa14","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-69tjt"}}
{"level":"info","ts":"2026-06-05T09:30:41.561Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"ef630fac-bff7-4ede-8704-66ee66152a5f","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:30:41.561Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"3e37b53a-96b1-45bc-8a51-e546cec2fa14","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:30:41.561Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"3e37b53a-96b1-45bc-8a51-e546cec2fa14","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:30:41.561Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"3e37b53a-96b1-45bc-8a51-e546cec2fa14","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:30:41.561Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"61743233-5a39-4bfe-8f5f-6c2df15b17ed"}
{"level":"info","ts":"2026-06-05T09:30:41.561Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"61743233-5a39-4bfe-8f5f-6c2df15b17ed"}
{"level":"info","ts":"2026-06-05T09:30:41.606Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"61743233-5a39-4bfe-8f5f-6c2df15b17ed","name":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-05T09:30:41.606Z","logger":"raycluster-resource","msg":"default","name":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:30:41.606Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"raycluster-auth","namespace":"test-ns-69tjt"}
{"level":"info","ts":"2026-06-05T09:30:41.705Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"b0df6b25-cc10-48b9-a229-1fd7b0ae42c0","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:30:41.706Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"ef630fac-bff7-4ede-8704-66ee66152a5f","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:30:41.706Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"142a04d6-fed3-43aa-a778-666c064c472f","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:30:41.905Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"142a04d6-fed3-43aa-a778-666c064c472f","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-06-05T09:30:41.906Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"b0df6b25-cc10-48b9-a229-1fd7b0ae42c0","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:30:42.006Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"3e37b53a-96b1-45bc-8a51-e546cec2fa14","name":"raycluster-auth-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:30:42.006Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"b0df6b25-cc10-48b9-a229-1fd7b0ae42c0","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:30:42.013Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"142a04d6-fed3-43aa-a778-666c064c472f","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-06-05T09:30:42.013Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"142a04d6-fed3-43aa-a778-666c064c472f","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:30:42.105Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"b0df6b25-cc10-48b9-a229-1fd7b0ae42c0"}
{"level":"info","ts":"2026-06-05T09:30:42.106Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"53b7117c-b662-42c3-a8f5-f282d48fcad6","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:30:42.106Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"b0df6b25-cc10-48b9-a229-1fd7b0ae42c0","rayCluster":"raycluster-auth","secret":"ca-secret"}
{"level":"info","ts":"2026-06-05T09:30:42.106Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"b0df6b25-cc10-48b9-a229-1fd7b0ae42c0","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:30:42.206Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"b0df6b25-cc10-48b9-a229-1fd7b0ae42c0","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:30:42.306Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"b0df6b25-cc10-48b9-a229-1fd7b0ae42c0"}
{"level":"info","ts":"2026-06-05T09:30:42.306Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"3e37b53a-96b1-45bc-8a51-e546cec2fa14","name":"kuberay-gateway-access","namespace":"test-ns-69tjt","operation":"created"}
{"level":"info","ts":"2026-06-05T09:30:42.306Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"53b7117c-b662-42c3-a8f5-f282d48fcad6","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-06-05T09:30:42.306Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"b0df6b25-cc10-48b9-a229-1fd7b0ae42c0","rayCluster":"raycluster-auth","podIPs":[]}
{"level":"info","ts":"2026-06-05T09:30:42.406Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"b0df6b25-cc10-48b9-a229-1fd7b0ae42c0"}
{"level":"info","ts":"2026-06-05T09:30:42.406Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"b0df6b25-cc10-48b9-a229-1fd7b0ae42c0","rayCluster":"raycluster-auth","podIPs":[]}
{"level":"info","ts":"2026-06-05T09:30:42.406Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"61743233-5a39-4bfe-8f5f-6c2df15b17ed","error":"head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"error","ts":"2026-06-05T09:30:42.406Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"b0df6b25-cc10-48b9-a229-1fd7b0ae42c0","error":"Certificate.cert-manager.io \"ray-worker-cert-raycluster-auth\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-05T09:30:42.406Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"61743233-5a39-4bfe-8f5f-6c2df15b17ed","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:30:42.406Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"b0df6b25-cc10-48b9-a229-1fd7b0ae42c0"}
{"level":"info","ts":"2026-06-05T09:30:42.406Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"61743233-5a39-4bfe-8f5f-6c2df15b17ed","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"error","ts":"2026-06-05T09:30:42.406Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"b0df6b25-cc10-48b9-a229-1fd7b0ae42c0","error":"Certificate.cert-manager.io \"ray-worker-cert-raycluster-auth\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:30:42.406Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"61743233-5a39-4bfe-8f5f-6c2df15b17ed","filter labels":["test-ns-69tjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:30:42.406Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"668363fd-8efe-4c43-9aac-deff840f235c","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:30:42.406Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"61743233-5a39-4bfe-8f5f-6c2df15b17ed","filter labels":["test-ns-69tjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:30:42.406Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"668363fd-8efe-4c43-9aac-deff840f235c"}
{"level":"info","ts":"2026-06-05T09:30:42.406Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"61743233-5a39-4bfe-8f5f-6c2df15b17ed","filter labels":["test-ns-69tjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:30:42.406Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"61743233-5a39-4bfe-8f5f-6c2df15b17ed","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-05T09:30:42.406Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"61743233-5a39-4bfe-8f5f-6c2df15b17ed","name":"raycluster-auth","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:30:42Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:30:42.411Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"288eb7db-318c-4b36-b230-89625155a28e","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:30:42.411Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"53b7117c-b662-42c3-a8f5-f282d48fcad6","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-06-05T09:30:42.411Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"53b7117c-b662-42c3-a8f5-f282d48fcad6","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:30:42.411Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"288eb7db-318c-4b36-b230-89625155a28e"}
{"level":"info","ts":"2026-06-05T09:30:42.412Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"61743233-5a39-4bfe-8f5f-6c2df15b17ed","name":"raycluster-auth","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-69tjt","name":"raycluster-auth"}}
{"level":"info","ts":"2026-06-05T09:30:42.412Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"61743233-5a39-4bfe-8f5f-6c2df15b17ed"}
{"level":"error","ts":"2026-06-05T09:30:42.412Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"61743233-5a39-4bfe-8f5f-6c2df15b17ed","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:30:42.412Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"56c024b6-5c49-49a4-b301-49844058ef22"}
{"level":"info","ts":"2026-06-05T09:30:42.412Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"56c024b6-5c49-49a4-b301-49844058ef22"}
{"level":"info","ts":"2026-06-05T09:30:42.412Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"56c024b6-5c49-49a4-b301-49844058ef22","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-05T09:30:42.412Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"56c024b6-5c49-49a4-b301-49844058ef22","error":"head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"error","ts":"2026-06-05T09:30:42.412Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"56c024b6-5c49-49a4-b301-49844058ef22","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:30:42.412Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"56c024b6-5c49-49a4-b301-49844058ef22","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"info","ts":"2026-06-05T09:30:42.412Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"56c024b6-5c49-49a4-b301-49844058ef22","filter labels":["test-ns-69tjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:30:42.412Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"56c024b6-5c49-49a4-b301-49844058ef22","filter labels":["test-ns-69tjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:30:42.412Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"56c024b6-5c49-49a4-b301-49844058ef22","filter labels":["test-ns-69tjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:30:42.412Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"56c024b6-5c49-49a4-b301-49844058ef22","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-05T09:30:42.412Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"56c024b6-5c49-49a4-b301-49844058ef22","name":"raycluster-auth","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:30:42Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:30:42.424Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"56c024b6-5c49-49a4-b301-49844058ef22"}
{"level":"error","ts":"2026-06-05T09:30:42.424Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"56c024b6-5c49-49a4-b301-49844058ef22","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:30:42.424Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"37248b22-f840-4cdf-ab36-1857a6a7ce51"}
{"level":"info","ts":"2026-06-05T09:30:42.424Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"37248b22-f840-4cdf-ab36-1857a6a7ce51"}
{"level":"info","ts":"2026-06-05T09:30:42.424Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"37248b22-f840-4cdf-ab36-1857a6a7ce51","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-05T09:30:42.424Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"37248b22-f840-4cdf-ab36-1857a6a7ce51","error":"head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"error","ts":"2026-06-05T09:30:42.424Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"37248b22-f840-4cdf-ab36-1857a6a7ce51","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:30:42.424Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"3e50657d-31bf-46bc-86e9-668743a85132","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:30:42.424Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"07cea298-10f9-472e-9b7a-b9aa52251ae7","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:30:42.424Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"37248b22-f840-4cdf-ab36-1857a6a7ce51","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"info","ts":"2026-06-05T09:30:42.424Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"37248b22-f840-4cdf-ab36-1857a6a7ce51","filter labels":["test-ns-69tjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:30:42.424Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"37248b22-f840-4cdf-ab36-1857a6a7ce51","filter labels":["test-ns-69tjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:30:42.424Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"37248b22-f840-4cdf-ab36-1857a6a7ce51","filter labels":["test-ns-69tjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:30:42.424Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"37248b22-f840-4cdf-ab36-1857a6a7ce51","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-05T09:30:42.424Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"37248b22-f840-4cdf-ab36-1857a6a7ce51","name":"raycluster-auth","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:30:42Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:30:42.424Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"3e50657d-31bf-46bc-86e9-668743a85132"}
{"level":"info","ts":"2026-06-05T09:30:42.430Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"37248b22-f840-4cdf-ab36-1857a6a7ce51","name":"raycluster-auth","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-69tjt","name":"raycluster-auth"}}
{"level":"info","ts":"2026-06-05T09:30:42.430Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"37248b22-f840-4cdf-ab36-1857a6a7ce51"}
{"level":"error","ts":"2026-06-05T09:30:42.430Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"37248b22-f840-4cdf-ab36-1857a6a7ce51","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:30:42.434Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"07cea298-10f9-472e-9b7a-b9aa52251ae7","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-06-05T09:30:42.447Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"07cea298-10f9-472e-9b7a-b9aa52251ae7","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-06-05T09:30:42.447Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"07cea298-10f9-472e-9b7a-b9aa52251ae7","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:30:42.505Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"a5b62af3-c982-498b-a70a-948cf8c1497c"}
{"level":"info","ts":"2026-06-05T09:30:42.505Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"a5b62af3-c982-498b-a70a-948cf8c1497c"}
{"level":"info","ts":"2026-06-05T09:30:42.506Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"a5b62af3-c982-498b-a70a-948cf8c1497c","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-05T09:30:42.506Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"a5b62af3-c982-498b-a70a-948cf8c1497c","error":"head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"error","ts":"2026-06-05T09:30:42.506Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"a5b62af3-c982-498b-a70a-948cf8c1497c","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:30:42.506Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"a5b62af3-c982-498b-a70a-948cf8c1497c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"info","ts":"2026-06-05T09:30:42.506Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"a5b62af3-c982-498b-a70a-948cf8c1497c","filter labels":["test-ns-69tjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:30:42.506Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"a5b62af3-c982-498b-a70a-948cf8c1497c","filter labels":["test-ns-69tjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:30:42.506Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"a5b62af3-c982-498b-a70a-948cf8c1497c","filter labels":["test-ns-69tjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:30:42.506Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"a5b62af3-c982-498b-a70a-948cf8c1497c"}
{"level":"error","ts":"2026-06-05T09:30:42.506Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"a5b62af3-c982-498b-a70a-948cf8c1497c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:30:42.526Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"3e37b53a-96b1-45bc-8a51-e546cec2fa14","name":"test-ns-69tjt-raycluster-auth","operation":"created"}
{"level":"info","ts":"2026-06-05T09:30:42.546Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"1c763537-137e-4318-840c-19baad66ef5b"}
{"level":"info","ts":"2026-06-05T09:30:42.546Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"1c763537-137e-4318-840c-19baad66ef5b"}
{"level":"info","ts":"2026-06-05T09:30:42.546Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"1c763537-137e-4318-840c-19baad66ef5b","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-05T09:30:42.546Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"1c763537-137e-4318-840c-19baad66ef5b","error":"head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"error","ts":"2026-06-05T09:30:42.546Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"1c763537-137e-4318-840c-19baad66ef5b","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:30:42.546Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"1c763537-137e-4318-840c-19baad66ef5b","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"info","ts":"2026-06-05T09:30:42.546Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"1c763537-137e-4318-840c-19baad66ef5b","filter labels":["test-ns-69tjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:30:42.546Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"1c763537-137e-4318-840c-19baad66ef5b","filter labels":["test-ns-69tjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:30:42.546Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"1c763537-137e-4318-840c-19baad66ef5b","filter labels":["test-ns-69tjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:30:42.546Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"1c763537-137e-4318-840c-19baad66ef5b"}
{"level":"error","ts":"2026-06-05T09:30:42.546Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"1c763537-137e-4318-840c-19baad66ef5b","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:30:42.705Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"54361bef-ed5c-4831-b1ce-306c7537b9cd"}
{"level":"info","ts":"2026-06-05T09:30:42.705Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"54361bef-ed5c-4831-b1ce-306c7537b9cd"}
{"level":"info","ts":"2026-06-05T09:30:42.706Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"54361bef-ed5c-4831-b1ce-306c7537b9cd","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-05T09:30:42.706Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"54361bef-ed5c-4831-b1ce-306c7537b9cd","error":"head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"error","ts":"2026-06-05T09:30:42.706Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"54361bef-ed5c-4831-b1ce-306c7537b9cd","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:30:42.706Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"54361bef-ed5c-4831-b1ce-306c7537b9cd","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"info","ts":"2026-06-05T09:30:42.706Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"54361bef-ed5c-4831-b1ce-306c7537b9cd","filter labels":["test-ns-69tjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:30:42.706Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"54361bef-ed5c-4831-b1ce-306c7537b9cd","filter labels":["test-ns-69tjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:30:42.706Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"54361bef-ed5c-4831-b1ce-306c7537b9cd","filter labels":["test-ns-69tjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:30:42.706Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"54361bef-ed5c-4831-b1ce-306c7537b9cd"}
{"level":"error","ts":"2026-06-05T09:30:42.706Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"54361bef-ed5c-4831-b1ce-306c7537b9cd","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:30:42.906Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"ad161bde-02c2-4d49-8a86-84ab34945669"}
{"level":"info","ts":"2026-06-05T09:30:42.906Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"ad161bde-02c2-4d49-8a86-84ab34945669"}
{"level":"info","ts":"2026-06-05T09:30:42.906Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"ad161bde-02c2-4d49-8a86-84ab34945669","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-05T09:30:42.906Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"ad161bde-02c2-4d49-8a86-84ab34945669","error":"head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"error","ts":"2026-06-05T09:30:42.906Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"ad161bde-02c2-4d49-8a86-84ab34945669","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:30:42.906Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"ad161bde-02c2-4d49-8a86-84ab34945669","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"info","ts":"2026-06-05T09:30:42.906Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"ad161bde-02c2-4d49-8a86-84ab34945669","filter labels":["test-ns-69tjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:30:42.906Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"ad161bde-02c2-4d49-8a86-84ab34945669","filter labels":["test-ns-69tjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:30:42.906Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"ad161bde-02c2-4d49-8a86-84ab34945669","filter labels":["test-ns-69tjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:30:42.906Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"ad161bde-02c2-4d49-8a86-84ab34945669"}
{"level":"error","ts":"2026-06-05T09:30:42.906Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"ad161bde-02c2-4d49-8a86-84ab34945669","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:30:43.227Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"5ef39e25-79a1-4d7e-a658-4c4e2b05e195"}
{"level":"info","ts":"2026-06-05T09:30:43.227Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"5ef39e25-79a1-4d7e-a658-4c4e2b05e195"}
{"level":"info","ts":"2026-06-05T09:30:43.227Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"5ef39e25-79a1-4d7e-a658-4c4e2b05e195","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-05T09:30:43.227Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"5ef39e25-79a1-4d7e-a658-4c4e2b05e195","error":"head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"error","ts":"2026-06-05T09:30:43.227Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"5ef39e25-79a1-4d7e-a658-4c4e2b05e195","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:30:43.227Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"5ef39e25-79a1-4d7e-a658-4c4e2b05e195","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"info","ts":"2026-06-05T09:30:43.227Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"5ef39e25-79a1-4d7e-a658-4c4e2b05e195","filter labels":["test-ns-69tjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:30:43.227Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"5ef39e25-79a1-4d7e-a658-4c4e2b05e195","filter labels":["test-ns-69tjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:30:43.227Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"5ef39e25-79a1-4d7e-a658-4c4e2b05e195","filter labels":["test-ns-69tjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:30:43.227Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"5ef39e25-79a1-4d7e-a658-4c4e2b05e195"}
{"level":"error","ts":"2026-06-05T09:30:43.227Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"5ef39e25-79a1-4d7e-a658-4c4e2b05e195","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:30:43.620Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"3e37b53a-96b1-45bc-8a51-e546cec2fa14","name":"kube-rbac-proxy-config-raycluster-auth","operation":"created"}
{"level":"error","ts":"2026-06-05T09:30:43.626Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"3e37b53a-96b1-45bc-8a51-e546cec2fa14","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:30:43.626Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"3e37b53a-96b1-45bc-8a51-e546cec2fa14"}
{"level":"error","ts":"2026-06-05T09:30:43.626Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"3e37b53a-96b1-45bc-8a51-e546cec2fa14","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:30:43.626Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"63cfe096-1549-41dd-88f0-042e0f36614a","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-69tjt"}}
{"level":"info","ts":"2026-06-05T09:30:43.626Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"63cfe096-1549-41dd-88f0-042e0f36614a","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:30:43.626Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"63cfe096-1549-41dd-88f0-042e0f36614a","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:30:43.642Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"63cfe096-1549-41dd-88f0-042e0f36614a","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:30:43.642Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"aa4418a8-75b3-452c-904b-7ddd1492652e","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-69tjt"}}
{"level":"info","ts":"2026-06-05T09:30:43.642Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"07164d81-2003-4055-a943-0d1bd1b0bcee","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:30:43.642Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"aa4418a8-75b3-452c-904b-7ddd1492652e","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:30:43.642Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"ebeb5771-ea4a-4f88-a891-1ede22cd6c3b","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:30:43.642Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"aa4418a8-75b3-452c-904b-7ddd1492652e","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:30:43.642Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"07164d81-2003-4055-a943-0d1bd1b0bcee"}
{"level":"info","ts":"2026-06-05T09:30:43.653Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"aa4418a8-75b3-452c-904b-7ddd1492652e","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:30:43.661Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"ebeb5771-ea4a-4f88-a891-1ede22cd6c3b","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-06-05T09:30:43.674Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"ebeb5771-ea4a-4f88-a891-1ede22cd6c3b","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-06-05T09:30:43.674Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"ebeb5771-ea4a-4f88-a891-1ede22cd6c3b","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:30:43.868Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0c5b2c5e-d2ce-4f29-ac17-486e126f3bc5"}
{"level":"info","ts":"2026-06-05T09:30:43.868Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0c5b2c5e-d2ce-4f29-ac17-486e126f3bc5"}
{"level":"info","ts":"2026-06-05T09:30:43.868Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0c5b2c5e-d2ce-4f29-ac17-486e126f3bc5","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-05T09:30:43.868Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0c5b2c5e-d2ce-4f29-ac17-486e126f3bc5"}
{"level":"info","ts":"2026-06-05T09:30:43.868Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0c5b2c5e-d2ce-4f29-ac17-486e126f3bc5","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:30:43.868Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0c5b2c5e-d2ce-4f29-ac17-486e126f3bc5"}
{"level":"info","ts":"2026-06-05T09:30:43.868Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0c5b2c5e-d2ce-4f29-ac17-486e126f3bc5","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:30:43.868Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0c5b2c5e-d2ce-4f29-ac17-486e126f3bc5","cluster":"raycluster-auth","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:30:43.868Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0c5b2c5e-d2ce-4f29-ac17-486e126f3bc5","cluster":"raycluster-auth","authType":"IntegratedOAuth","serviceAccount":"raycluster-auth-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-05T09:30:43.868Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0c5b2c5e-d2ce-4f29-ac17-486e126f3bc5","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"raycluster-auth","ray.io/group":"headgroup","ray.io/identifier":"raycluster-auth-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-05T09:30:43.868Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0c5b2c5e-d2ce-4f29-ac17-486e126f3bc5","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"raycluster-auth-head-svc.test-ns-69tjt.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-05T09:30:43.869Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0c5b2c5e-d2ce-4f29-ac17-486e126f3bc5","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-69tjt.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-05T09:30:43.869Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0c5b2c5e-d2ce-4f29-ac17-486e126f3bc5","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-69tjt.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-05T09:30:43.869Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0c5b2c5e-d2ce-4f29-ac17-486e126f3bc5","enabled":true}
{"level":"info","ts":"2026-06-05T09:30:43.888Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0c5b2c5e-d2ce-4f29-ac17-486e126f3bc5","name":"raycluster-auth-head-k2fq2"}
{"level":"info","ts":"2026-06-05T09:30:43.888Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0c5b2c5e-d2ce-4f29-ac17-486e126f3bc5","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:30:43.888Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0c5b2c5e-d2ce-4f29-ac17-486e126f3bc5","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:30:43.888Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0c5b2c5e-d2ce-4f29-ac17-486e126f3bc5","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-05T09:30:43.888Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0c5b2c5e-d2ce-4f29-ac17-486e126f3bc5","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-05T09:30:43.888Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0c5b2c5e-d2ce-4f29-ac17-486e126f3bc5","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-05T09:30:43.888Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0c5b2c5e-d2ce-4f29-ac17-486e126f3bc5"}
{"level":"info","ts":"2026-06-05T09:30:43.888Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0c5b2c5e-d2ce-4f29-ac17-486e126f3bc5","nodeType":"worker","rayStartParams":{"address":"raycluster-auth-head-svc.test-ns-69tjt.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-05T09:30:43.888Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0c5b2c5e-d2ce-4f29-ac17-486e126f3bc5","rayStartCmd":"ray start  --address=raycluster-auth-head-svc.test-ns-69tjt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:30:43.888Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0c5b2c5e-d2ce-4f29-ac17-486e126f3bc5","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-auth-head-svc.test-ns-69tjt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:30:43.888Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0c5b2c5e-d2ce-4f29-ac17-486e126f3bc5","enabled":true}
{"level":"info","ts":"2026-06-05T09:30:43.903Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0c5b2c5e-d2ce-4f29-ac17-486e126f3bc5","name":"raycluster-auth-small-group-worker-w5wpq"}
{"level":"info","ts":"2026-06-05T09:30:43.904Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0c5b2c5e-d2ce-4f29-ac17-486e126f3bc5","podName":"raycluster-auth-head-k2fq2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:30:43.904Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0c5b2c5e-d2ce-4f29-ac17-486e126f3bc5","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"raycluster-auth-head-svc"},"newHead":{"podName":"raycluster-auth-head-k2fq2","serviceName":"raycluster-auth-head-svc"}}
{"level":"info","ts":"2026-06-05T09:30:43.904Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0c5b2c5e-d2ce-4f29-ac17-486e126f3bc5","name":"raycluster-auth","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:30:42Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:30:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:30:43Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-auth-head-k2fq2","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:30:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:30:43.925Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"b4c81c2b-20ce-4df5-9fe8-02e7ca4d029b"}
{"level":"info","ts":"2026-06-05T09:30:43.925Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"b4c81c2b-20ce-4df5-9fe8-02e7ca4d029b"}
{"level":"info","ts":"2026-06-05T09:30:43.925Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"b4c81c2b-20ce-4df5-9fe8-02e7ca4d029b","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-05T09:30:43.925Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"8d3f8e67-87d6-497a-8194-c437ac2f54bf","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:30:43.926Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"b4c81c2b-20ce-4df5-9fe8-02e7ca4d029b","Found 1 head Pod":"raycluster-auth-head-k2fq2","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:30:43.926Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"b4c81c2b-20ce-4df5-9fe8-02e7ca4d029b","head Pod":"raycluster-auth-head-k2fq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-k2fq2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:30:43.926Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"b4c81c2b-20ce-4df5-9fe8-02e7ca4d029b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:30:43.926Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"8d3f8e67-87d6-497a-8194-c437ac2f54bf"}
{"level":"info","ts":"2026-06-05T09:30:43.926Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"b4c81c2b-20ce-4df5-9fe8-02e7ca4d029b","worker Pod":"raycluster-auth-small-group-worker-w5wpq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-w5wpq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:30:43.926Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"b4c81c2b-20ce-4df5-9fe8-02e7ca4d029b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:30:43.926Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"b4c81c2b-20ce-4df5-9fe8-02e7ca4d029b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:30:43.926Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"b4c81c2b-20ce-4df5-9fe8-02e7ca4d029b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:30:43.925Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"4ebea0fe-32c2-4b8c-8262-f7d3164603c7","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:30:43.926Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"b4c81c2b-20ce-4df5-9fe8-02e7ca4d029b","podName":"raycluster-auth-head-k2fq2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:30:43.926Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"b4c81c2b-20ce-4df5-9fe8-02e7ca4d029b","podName":"raycluster-auth-head-k2fq2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:30:43.926Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"b4c81c2b-20ce-4df5-9fe8-02e7ca4d029b","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"raycluster-auth-head-svc"},"newHead":{"podName":"raycluster-auth-head-k2fq2","serviceName":"raycluster-auth-head-svc"}}
{"level":"info","ts":"2026-06-05T09:30:43.926Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"b4c81c2b-20ce-4df5-9fe8-02e7ca4d029b","name":"raycluster-auth","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:30:42Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:30:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:30:43Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-auth-head-k2fq2","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:30:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:30:43.949Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"b4c81c2b-20ce-4df5-9fe8-02e7ca4d029b","name":"raycluster-auth","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-69tjt","name":"raycluster-auth"}}
{"level":"info","ts":"2026-06-05T09:30:43.949Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"b4c81c2b-20ce-4df5-9fe8-02e7ca4d029b"}
{"level":"error","ts":"2026-06-05T09:30:43.949Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"b4c81c2b-20ce-4df5-9fe8-02e7ca4d029b","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:30:43.949Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0ee9da69-cf24-420c-a48b-d4bf26e3acf8"}
{"level":"info","ts":"2026-06-05T09:30:43.949Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0ee9da69-cf24-420c-a48b-d4bf26e3acf8"}
{"level":"info","ts":"2026-06-05T09:30:43.950Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0ee9da69-cf24-420c-a48b-d4bf26e3acf8","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-05T09:30:43.950Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0ee9da69-cf24-420c-a48b-d4bf26e3acf8","Found 1 head Pod":"raycluster-auth-head-k2fq2","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:30:43.950Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0ee9da69-cf24-420c-a48b-d4bf26e3acf8","head Pod":"raycluster-auth-head-k2fq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-k2fq2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:30:43.950Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0ee9da69-cf24-420c-a48b-d4bf26e3acf8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:30:43.950Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0ee9da69-cf24-420c-a48b-d4bf26e3acf8","worker Pod":"raycluster-auth-small-group-worker-w5wpq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-w5wpq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:30:43.950Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0ee9da69-cf24-420c-a48b-d4bf26e3acf8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:30:43.950Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0ee9da69-cf24-420c-a48b-d4bf26e3acf8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:30:43.950Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0ee9da69-cf24-420c-a48b-d4bf26e3acf8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:30:43.950Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0ee9da69-cf24-420c-a48b-d4bf26e3acf8","podName":"raycluster-auth-head-k2fq2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:30:43.950Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0ee9da69-cf24-420c-a48b-d4bf26e3acf8","podName":"raycluster-auth-head-k2fq2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:30:43.950Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0ee9da69-cf24-420c-a48b-d4bf26e3acf8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:30:43.950Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0ee9da69-cf24-420c-a48b-d4bf26e3acf8","seconds":300}
{"level":"info","ts":"2026-06-05T09:30:43.955Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4008e1ee-22d9-48ce-aa17-d02a09b0d2d7"}
{"level":"info","ts":"2026-06-05T09:30:43.955Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4008e1ee-22d9-48ce-aa17-d02a09b0d2d7"}
{"level":"info","ts":"2026-06-05T09:30:43.955Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4008e1ee-22d9-48ce-aa17-d02a09b0d2d7","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-05T09:30:43.955Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4008e1ee-22d9-48ce-aa17-d02a09b0d2d7","Found 1 head Pod":"raycluster-auth-head-k2fq2","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:30:43.955Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4008e1ee-22d9-48ce-aa17-d02a09b0d2d7","head Pod":"raycluster-auth-head-k2fq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-k2fq2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:30:43.955Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4008e1ee-22d9-48ce-aa17-d02a09b0d2d7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:30:43.955Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4008e1ee-22d9-48ce-aa17-d02a09b0d2d7","worker Pod":"raycluster-auth-small-group-worker-w5wpq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-w5wpq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:30:43.955Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4008e1ee-22d9-48ce-aa17-d02a09b0d2d7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:30:43.955Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4008e1ee-22d9-48ce-aa17-d02a09b0d2d7","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:30:43.955Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4008e1ee-22d9-48ce-aa17-d02a09b0d2d7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:30:43.955Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4008e1ee-22d9-48ce-aa17-d02a09b0d2d7","podName":"raycluster-auth-head-k2fq2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:30:43.955Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4008e1ee-22d9-48ce-aa17-d02a09b0d2d7","podName":"raycluster-auth-head-k2fq2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:30:43.956Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4008e1ee-22d9-48ce-aa17-d02a09b0d2d7","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:30:43.956Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4008e1ee-22d9-48ce-aa17-d02a09b0d2d7","seconds":300}
{"level":"info","ts":"2026-06-05T09:30:43.956Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"4ebea0fe-32c2-4b8c-8262-f7d3164603c7","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-06-05T09:30:43.957Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"1a729b48-6be5-47ad-94be-d5ce255a5935"}
{"level":"info","ts":"2026-06-05T09:30:43.957Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"1a729b48-6be5-47ad-94be-d5ce255a5935"}
{"level":"info","ts":"2026-06-05T09:30:43.957Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"1a729b48-6be5-47ad-94be-d5ce255a5935","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-05T09:30:43.957Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"1a729b48-6be5-47ad-94be-d5ce255a5935","Found 1 head Pod":"raycluster-auth-head-k2fq2","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:30:43.957Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"1a729b48-6be5-47ad-94be-d5ce255a5935","head Pod":"raycluster-auth-head-k2fq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-k2fq2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:30:43.957Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"1a729b48-6be5-47ad-94be-d5ce255a5935","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:30:43.957Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"1a729b48-6be5-47ad-94be-d5ce255a5935","worker Pod":"raycluster-auth-small-group-worker-w5wpq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-w5wpq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:30:43.957Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"1a729b48-6be5-47ad-94be-d5ce255a5935","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:30:43.957Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"1a729b48-6be5-47ad-94be-d5ce255a5935","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:30:43.957Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"1a729b48-6be5-47ad-94be-d5ce255a5935","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:30:43.957Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"1a729b48-6be5-47ad-94be-d5ce255a5935","podName":"raycluster-auth-head-k2fq2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:30:43.957Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"1a729b48-6be5-47ad-94be-d5ce255a5935","podName":"raycluster-auth-head-k2fq2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:30:43.957Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"1a729b48-6be5-47ad-94be-d5ce255a5935","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:30:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:30:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-05T09:30:43.957Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"1a729b48-6be5-47ad-94be-d5ce255a5935","name":"raycluster-auth","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:30:43Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-auth-head-k2fq2","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:30:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:30:43Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-auth-head-k2fq2","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:30:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:30:44.006Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"efd0819a-51ea-442f-a4f2-4fad3e1c842f"}
{"level":"info","ts":"2026-06-05T09:30:44.006Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"efd0819a-51ea-442f-a4f2-4fad3e1c842f"}
{"level":"info","ts":"2026-06-05T09:30:44.006Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"cbdf0875-d2e8-4bf3-9da8-d2aa923615c4","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:30:44.006Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"efd0819a-51ea-442f-a4f2-4fad3e1c842f","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-05T09:30:44.006Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"efd0819a-51ea-442f-a4f2-4fad3e1c842f","Found 1 head Pod":"raycluster-auth-head-k2fq2","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:30:44.006Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"efd0819a-51ea-442f-a4f2-4fad3e1c842f","head Pod":"raycluster-auth-head-k2fq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-k2fq2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:30:44.006Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"efd0819a-51ea-442f-a4f2-4fad3e1c842f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:30:44.006Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"cbdf0875-d2e8-4bf3-9da8-d2aa923615c4"}
{"level":"info","ts":"2026-06-05T09:30:44.006Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"efd0819a-51ea-442f-a4f2-4fad3e1c842f","worker Pod":"raycluster-auth-small-group-worker-w5wpq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-w5wpq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:30:44.006Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"efd0819a-51ea-442f-a4f2-4fad3e1c842f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:30:44.006Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"efd0819a-51ea-442f-a4f2-4fad3e1c842f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:30:44.006Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"efd0819a-51ea-442f-a4f2-4fad3e1c842f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:30:44.006Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"efd0819a-51ea-442f-a4f2-4fad3e1c842f","podName":"raycluster-auth-head-k2fq2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:30:44.006Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"efd0819a-51ea-442f-a4f2-4fad3e1c842f","podName":"raycluster-auth-head-k2fq2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:30:44.007Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"efd0819a-51ea-442f-a4f2-4fad3e1c842f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:30:44.007Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"efd0819a-51ea-442f-a4f2-4fad3e1c842f","seconds":300}
{"level":"info","ts":"2026-06-05T09:30:44.011Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"4ebea0fe-32c2-4b8c-8262-f7d3164603c7","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-06-05T09:30:44.011Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"4ebea0fe-32c2-4b8c-8262-f7d3164603c7","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:30:44.011Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"dacd9f59-539f-434b-8cc5-cc11c51de745","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:30:44.025Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"dacd9f59-539f-434b-8cc5-cc11c51de745","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-06-05T09:30:44.037Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"dacd9f59-539f-434b-8cc5-cc11c51de745","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-06-05T09:30:44.037Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"dacd9f59-539f-434b-8cc5-cc11c51de745","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:30:44.336Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"9abfb565-baae-4b45-952b-19a2bf26c757"}
{"level":"info","ts":"2026-06-05T09:30:44.336Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"9abfb565-baae-4b45-952b-19a2bf26c757"}
{"level":"info","ts":"2026-06-05T09:30:44.336Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"9abfb565-baae-4b45-952b-19a2bf26c757","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-05T09:30:44.336Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"9abfb565-baae-4b45-952b-19a2bf26c757","Found 1 head Pod":"raycluster-auth-head-k2fq2","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:30:44.337Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"9abfb565-baae-4b45-952b-19a2bf26c757","head Pod":"raycluster-auth-head-k2fq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-k2fq2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:30:44.337Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"9abfb565-baae-4b45-952b-19a2bf26c757","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:30:44.337Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"9abfb565-baae-4b45-952b-19a2bf26c757","worker Pod":"raycluster-auth-small-group-worker-w5wpq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-w5wpq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:30:44.337Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"9abfb565-baae-4b45-952b-19a2bf26c757","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:30:44.337Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"9abfb565-baae-4b45-952b-19a2bf26c757","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:30:44.337Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"9abfb565-baae-4b45-952b-19a2bf26c757","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:30:44.337Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"9abfb565-baae-4b45-952b-19a2bf26c757","podName":"raycluster-auth-small-group-worker-w5wpq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:30:44.337Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"9abfb565-baae-4b45-952b-19a2bf26c757","podName":"raycluster-auth-small-group-worker-w5wpq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:30:44.337Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"9abfb565-baae-4b45-952b-19a2bf26c757","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:30:44.337Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"9abfb565-baae-4b45-952b-19a2bf26c757","seconds":300}
{"level":"info","ts":"2026-06-05T09:30:44.364Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"07868cc3-c170-4a24-bd97-d6e07ea883f9"}
{"level":"info","ts":"2026-06-05T09:30:44.364Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"07868cc3-c170-4a24-bd97-d6e07ea883f9"}
{"level":"info","ts":"2026-06-05T09:30:44.364Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"07868cc3-c170-4a24-bd97-d6e07ea883f9","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-05T09:30:44.365Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"07868cc3-c170-4a24-bd97-d6e07ea883f9","Found 1 head Pod":"raycluster-auth-head-k2fq2","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:30:44.365Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"07868cc3-c170-4a24-bd97-d6e07ea883f9","head Pod":"raycluster-auth-head-k2fq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-k2fq2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:30:44.365Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"07868cc3-c170-4a24-bd97-d6e07ea883f9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:30:44.365Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"07868cc3-c170-4a24-bd97-d6e07ea883f9","worker Pod":"raycluster-auth-small-group-worker-w5wpq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-w5wpq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:30:44.365Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"07868cc3-c170-4a24-bd97-d6e07ea883f9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:30:44.365Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"07868cc3-c170-4a24-bd97-d6e07ea883f9","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:30:44.365Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"07868cc3-c170-4a24-bd97-d6e07ea883f9","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:30:44.365Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"07868cc3-c170-4a24-bd97-d6e07ea883f9","podName":"raycluster-auth-head-k2fq2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:30:44.365Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"07868cc3-c170-4a24-bd97-d6e07ea883f9","podName":"raycluster-auth-head-k2fq2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:30:44.365Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"07868cc3-c170-4a24-bd97-d6e07ea883f9","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:30:44.365Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"07868cc3-c170-4a24-bd97-d6e07ea883f9","seconds":300}
{"level":"info","ts":"2026-06-05T09:30:46.007Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"46b403cd-068e-436b-a92d-154da3f9202c"}
{"level":"info","ts":"2026-06-05T09:30:46.007Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"46b403cd-068e-436b-a92d-154da3f9202c"}
{"level":"info","ts":"2026-06-05T09:30:46.007Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"46b403cd-068e-436b-a92d-154da3f9202c","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-05T09:30:46.007Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"46b403cd-068e-436b-a92d-154da3f9202c","Found 1 head Pod":"raycluster-auth-head-k2fq2","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:30:46.007Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"46b403cd-068e-436b-a92d-154da3f9202c","head Pod":"raycluster-auth-head-k2fq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-k2fq2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:30:46.007Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"46b403cd-068e-436b-a92d-154da3f9202c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:30:46.007Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"46b403cd-068e-436b-a92d-154da3f9202c","worker Pod":"raycluster-auth-small-group-worker-w5wpq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-w5wpq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:30:46.007Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"46b403cd-068e-436b-a92d-154da3f9202c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:30:46.007Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"46b403cd-068e-436b-a92d-154da3f9202c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:30:46.007Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"46b403cd-068e-436b-a92d-154da3f9202c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:30:46.007Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"46b403cd-068e-436b-a92d-154da3f9202c","podName":"raycluster-auth-head-k2fq2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:30:46.007Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"46b403cd-068e-436b-a92d-154da3f9202c","podName":"raycluster-auth-head-k2fq2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:30:46.007Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"46b403cd-068e-436b-a92d-154da3f9202c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:30:46.007Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"46b403cd-068e-436b-a92d-154da3f9202c","seconds":300}
{"level":"info","ts":"2026-06-05T09:31:12.412Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"0e005855-7d7d-4331-bbef-74404c02e640","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:31:12.412Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"0e005855-7d7d-4331-bbef-74404c02e640","rayCluster":"raycluster-auth","podIPs":[]}
{"level":"info","ts":"2026-06-05T09:32:12.413Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"110d3fbc-8119-448f-9b87-5ad59e191720","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:32:12.413Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"110d3fbc-8119-448f-9b87-5ad59e191720","rayCluster":"raycluster-auth","podIPs":[]}
{"level":"info","ts":"2026-06-05T09:33:12.414Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"587aaaee-1211-4c53-85f2-b5183f95b392","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:33:12.414Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"587aaaee-1211-4c53-85f2-b5183f95b392","rayCluster":"raycluster-auth","podIPs":[]}
{"level":"info","ts":"2026-06-05T09:34:12.414Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"23101724-127e-450d-8c35-c3c050a70690","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:34:12.415Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"23101724-127e-450d-8c35-c3c050a70690","rayCluster":"raycluster-auth","podIPs":[]}
{"level":"info","ts":"2026-06-05T09:34:44.809Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"fd9c1775-58a8-43da-b06b-746de63f8bbe"}
{"level":"info","ts":"2026-06-05T09:34:44.809Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"fd9c1775-58a8-43da-b06b-746de63f8bbe"}
{"level":"info","ts":"2026-06-05T09:34:44.809Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"fd9c1775-58a8-43da-b06b-746de63f8bbe","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-05T09:34:44.809Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"fd9c1775-58a8-43da-b06b-746de63f8bbe","Found 1 head Pod":"raycluster-auth-head-k2fq2","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:34:44.809Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"fd9c1775-58a8-43da-b06b-746de63f8bbe","head Pod":"raycluster-auth-head-k2fq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-k2fq2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:34:44.809Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"fd9c1775-58a8-43da-b06b-746de63f8bbe","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:34:44.809Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"fd9c1775-58a8-43da-b06b-746de63f8bbe","worker Pod":"raycluster-auth-small-group-worker-w5wpq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-w5wpq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:34:44.809Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"fd9c1775-58a8-43da-b06b-746de63f8bbe","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:34:44.809Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"fd9c1775-58a8-43da-b06b-746de63f8bbe","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:34:44.809Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"fd9c1775-58a8-43da-b06b-746de63f8bbe","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:34:44.810Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"fd9c1775-58a8-43da-b06b-746de63f8bbe","podName":"raycluster-auth-small-group-worker-w5wpq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:34:44.810Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"fd9c1775-58a8-43da-b06b-746de63f8bbe","podName":"raycluster-auth-small-group-worker-w5wpq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:34:44.810Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"fd9c1775-58a8-43da-b06b-746de63f8bbe","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:34:44.810Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"fd9c1775-58a8-43da-b06b-746de63f8bbe","seconds":300}
{"level":"info","ts":"2026-06-05T09:34:44.829Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0fe7b5f0-e68f-4492-8c52-1d78a9c398a8"}
{"level":"info","ts":"2026-06-05T09:34:44.829Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0fe7b5f0-e68f-4492-8c52-1d78a9c398a8"}
{"level":"info","ts":"2026-06-05T09:34:44.829Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0fe7b5f0-e68f-4492-8c52-1d78a9c398a8","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-05T09:34:44.829Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0fe7b5f0-e68f-4492-8c52-1d78a9c398a8","Found 1 head Pod":"raycluster-auth-head-k2fq2","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:34:44.829Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0fe7b5f0-e68f-4492-8c52-1d78a9c398a8","head Pod":"raycluster-auth-head-k2fq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-k2fq2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:34:44.829Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0fe7b5f0-e68f-4492-8c52-1d78a9c398a8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:34:44.829Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0fe7b5f0-e68f-4492-8c52-1d78a9c398a8","worker Pod":"raycluster-auth-small-group-worker-w5wpq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-w5wpq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:34:44.829Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0fe7b5f0-e68f-4492-8c52-1d78a9c398a8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:34:44.829Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0fe7b5f0-e68f-4492-8c52-1d78a9c398a8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:34:44.829Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0fe7b5f0-e68f-4492-8c52-1d78a9c398a8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:34:44.829Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0fe7b5f0-e68f-4492-8c52-1d78a9c398a8","podName":"raycluster-auth-head-k2fq2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:34:44.829Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0fe7b5f0-e68f-4492-8c52-1d78a9c398a8","podName":"raycluster-auth-head-k2fq2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:34:44.829Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0fe7b5f0-e68f-4492-8c52-1d78a9c398a8","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"raycluster-auth-head-k2fq2","serviceName":"raycluster-auth-head-svc"},"newHead":{"podIP":"10.132.0.14","serviceIP":"10.132.0.14","podName":"raycluster-auth-head-k2fq2","serviceName":"raycluster-auth-head-svc"}}
{"level":"info","ts":"2026-06-05T09:34:44.829Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0fe7b5f0-e68f-4492-8c52-1d78a9c398a8","name":"raycluster-auth","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:30:43Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-auth-head-k2fq2","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:30:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:34:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.14","serviceIP":"10.132.0.14","podName":"raycluster-auth-head-k2fq2","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:30:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:34:44.841Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"b269f880-cd06-4517-9e39-3398196d4a6d","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:34:44.841Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"af82a06c-cedd-4d5f-89d9-c28e91168f96","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:34:44.842Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"af82a06c-cedd-4d5f-89d9-c28e91168f96","pod":"raycluster-auth-head-k2fq2","ip":"10.132.0.14"}
{"level":"info","ts":"2026-06-05T09:34:44.842Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"af82a06c-cedd-4d5f-89d9-c28e91168f96","pod":"raycluster-auth-small-group-worker-w5wpq","ip":"10.133.0.34"}
{"level":"info","ts":"2026-06-05T09:34:44.858Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"b269f880-cd06-4517-9e39-3398196d4a6d","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-06-05T09:34:44.869Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"af82a06c-cedd-4d5f-89d9-c28e91168f96","certificate":"ray-head-cert-raycluster-auth","dnsNames":["raycluster-auth-head-svc","localhost","raycluster-auth-head-svc.test-ns-69tjt.svc","raycluster-auth-head-svc.test-ns-69tjt.svc.cluster.local"],"ipAddresses":["10.132.0.14","10.133.0.34","127.0.0.1"]}
{"level":"info","ts":"2026-06-05T09:34:44.871Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"b269f880-cd06-4517-9e39-3398196d4a6d","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-06-05T09:34:44.871Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"b269f880-cd06-4517-9e39-3398196d4a6d","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:34:44.881Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"af82a06c-cedd-4d5f-89d9-c28e91168f96","certificate":"ray-worker-cert-raycluster-auth","dnsNames":["raycluster-auth-worker-svc","localhost","raycluster-auth-small-group","raycluster-auth-worker-svc.test-ns-69tjt.svc","raycluster-auth-worker-svc.test-ns-69tjt.svc.cluster.local","*.raycluster-auth-worker-svc.test-ns-69tjt.svc","*.raycluster-auth-worker-svc.test-ns-69tjt.svc.cluster.local","*-worker-*.test-ns-69tjt.svc","*-worker-*.test-ns-69tjt.svc.cluster.local"],"ipAddresses":["10.132.0.14","10.133.0.34","127.0.0.1"]}
{"level":"info","ts":"2026-06-05T09:34:44.881Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"af82a06c-cedd-4d5f-89d9-c28e91168f96","rayCluster":"raycluster-auth","podIPs":["10.132.0.14","10.133.0.34"]}
{"level":"info","ts":"2026-06-05T09:34:46.842Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"56ff8d3b-8a56-48bf-afa1-0dc188b36ed1"}
{"level":"info","ts":"2026-06-05T09:34:46.842Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"56ff8d3b-8a56-48bf-afa1-0dc188b36ed1"}
{"level":"info","ts":"2026-06-05T09:34:46.842Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"56ff8d3b-8a56-48bf-afa1-0dc188b36ed1","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-05T09:34:46.842Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"56ff8d3b-8a56-48bf-afa1-0dc188b36ed1","Found 1 head Pod":"raycluster-auth-head-k2fq2","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:34:46.842Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"56ff8d3b-8a56-48bf-afa1-0dc188b36ed1","head Pod":"raycluster-auth-head-k2fq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-k2fq2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:34:46.842Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"56ff8d3b-8a56-48bf-afa1-0dc188b36ed1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:34:46.842Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"56ff8d3b-8a56-48bf-afa1-0dc188b36ed1","worker Pod":"raycluster-auth-small-group-worker-w5wpq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-w5wpq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:34:46.842Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"56ff8d3b-8a56-48bf-afa1-0dc188b36ed1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:34:46.842Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"56ff8d3b-8a56-48bf-afa1-0dc188b36ed1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:34:46.842Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"56ff8d3b-8a56-48bf-afa1-0dc188b36ed1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:34:46.842Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"56ff8d3b-8a56-48bf-afa1-0dc188b36ed1","podName":"raycluster-auth-small-group-worker-w5wpq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:34:46.842Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"56ff8d3b-8a56-48bf-afa1-0dc188b36ed1","podName":"raycluster-auth-small-group-worker-w5wpq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:34:46.842Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"56ff8d3b-8a56-48bf-afa1-0dc188b36ed1","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:34:46.842Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"56ff8d3b-8a56-48bf-afa1-0dc188b36ed1","seconds":300}
{"level":"info","ts":"2026-06-05T09:34:49.846Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4dd7df5c-7c2c-44ac-b192-3eb69dae55cc"}
{"level":"info","ts":"2026-06-05T09:34:49.846Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4dd7df5c-7c2c-44ac-b192-3eb69dae55cc"}
{"level":"info","ts":"2026-06-05T09:34:49.847Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4dd7df5c-7c2c-44ac-b192-3eb69dae55cc","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-05T09:34:49.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4dd7df5c-7c2c-44ac-b192-3eb69dae55cc","Found 1 head Pod":"raycluster-auth-head-k2fq2","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:34:49.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4dd7df5c-7c2c-44ac-b192-3eb69dae55cc","head Pod":"raycluster-auth-head-k2fq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-k2fq2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:34:49.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4dd7df5c-7c2c-44ac-b192-3eb69dae55cc","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:34:49.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4dd7df5c-7c2c-44ac-b192-3eb69dae55cc","worker Pod":"raycluster-auth-small-group-worker-w5wpq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-w5wpq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:34:49.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4dd7df5c-7c2c-44ac-b192-3eb69dae55cc","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:34:49.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4dd7df5c-7c2c-44ac-b192-3eb69dae55cc","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:34:49.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4dd7df5c-7c2c-44ac-b192-3eb69dae55cc","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:34:49.847Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4dd7df5c-7c2c-44ac-b192-3eb69dae55cc","podName":"raycluster-auth-small-group-worker-w5wpq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:34:49.847Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4dd7df5c-7c2c-44ac-b192-3eb69dae55cc","podName":"raycluster-auth-small-group-worker-w5wpq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:34:49.847Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4dd7df5c-7c2c-44ac-b192-3eb69dae55cc","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:34:49.847Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4dd7df5c-7c2c-44ac-b192-3eb69dae55cc","seconds":300}
{"level":"info","ts":"2026-06-05T09:34:53.841Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"5319fca6-096c-4e52-ad7d-2993b49248a2"}
{"level":"info","ts":"2026-06-05T09:34:53.841Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"5319fca6-096c-4e52-ad7d-2993b49248a2"}
{"level":"info","ts":"2026-06-05T09:34:53.841Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"5319fca6-096c-4e52-ad7d-2993b49248a2","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-05T09:34:53.841Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"5319fca6-096c-4e52-ad7d-2993b49248a2","Found 1 head Pod":"raycluster-auth-head-k2fq2","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:34:53.841Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"5319fca6-096c-4e52-ad7d-2993b49248a2","head Pod":"raycluster-auth-head-k2fq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-k2fq2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:34:53.841Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"5319fca6-096c-4e52-ad7d-2993b49248a2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:34:53.841Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"5319fca6-096c-4e52-ad7d-2993b49248a2","worker Pod":"raycluster-auth-small-group-worker-w5wpq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-w5wpq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:34:53.841Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"5319fca6-096c-4e52-ad7d-2993b49248a2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:34:53.841Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"5319fca6-096c-4e52-ad7d-2993b49248a2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:34:53.841Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"5319fca6-096c-4e52-ad7d-2993b49248a2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:34:53.841Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"5319fca6-096c-4e52-ad7d-2993b49248a2","podName":"raycluster-auth-small-group-worker-w5wpq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:34:53.841Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"5319fca6-096c-4e52-ad7d-2993b49248a2","podName":"raycluster-auth-small-group-worker-w5wpq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:34:53.841Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"5319fca6-096c-4e52-ad7d-2993b49248a2","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:34:53.841Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"5319fca6-096c-4e52-ad7d-2993b49248a2","seconds":300}
{"level":"info","ts":"2026-06-05T09:34:54.847Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"2e3be966-9d52-4f75-b692-a4b04a3ecd66"}
{"level":"info","ts":"2026-06-05T09:34:54.847Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"2e3be966-9d52-4f75-b692-a4b04a3ecd66"}
{"level":"info","ts":"2026-06-05T09:34:54.847Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"2e3be966-9d52-4f75-b692-a4b04a3ecd66","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-05T09:34:54.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"2e3be966-9d52-4f75-b692-a4b04a3ecd66","Found 1 head Pod":"raycluster-auth-head-k2fq2","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:34:54.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"2e3be966-9d52-4f75-b692-a4b04a3ecd66","head Pod":"raycluster-auth-head-k2fq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-k2fq2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:34:54.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"2e3be966-9d52-4f75-b692-a4b04a3ecd66","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:34:54.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"2e3be966-9d52-4f75-b692-a4b04a3ecd66","worker Pod":"raycluster-auth-small-group-worker-w5wpq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-w5wpq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:34:54.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"2e3be966-9d52-4f75-b692-a4b04a3ecd66","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:34:54.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"2e3be966-9d52-4f75-b692-a4b04a3ecd66","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:34:54.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"2e3be966-9d52-4f75-b692-a4b04a3ecd66","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:34:54.847Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"2e3be966-9d52-4f75-b692-a4b04a3ecd66","podName":"raycluster-auth-head-k2fq2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:34:54.847Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"2e3be966-9d52-4f75-b692-a4b04a3ecd66","podName":"raycluster-auth-head-k2fq2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:34:54.847Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"2e3be966-9d52-4f75-b692-a4b04a3ecd66","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-05T09:34:54.847Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"2e3be966-9d52-4f75-b692-a4b04a3ecd66","name":"raycluster-auth","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:34:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.14","serviceIP":"10.132.0.14","podName":"raycluster-auth-head-k2fq2","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:30:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:34:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.14","serviceIP":"10.132.0.14","podName":"raycluster-auth-head-k2fq2","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:30:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:34:54.860Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"532a1a09-f20c-44b0-a7a7-d7dbbee8a060","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:34:54.860Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"6c0fefbb-adee-4ac2-aa18-c9dc25e78a85","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:34:54.861Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"532a1a09-f20c-44b0-a7a7-d7dbbee8a060","pod":"raycluster-auth-head-k2fq2","ip":"10.132.0.14"}
{"level":"info","ts":"2026-06-05T09:34:54.861Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"532a1a09-f20c-44b0-a7a7-d7dbbee8a060","pod":"raycluster-auth-small-group-worker-w5wpq","ip":"10.133.0.34"}
{"level":"info","ts":"2026-06-05T09:34:54.861Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"532a1a09-f20c-44b0-a7a7-d7dbbee8a060","rayCluster":"raycluster-auth","podIPs":["10.132.0.14","10.133.0.34"]}
{"level":"info","ts":"2026-06-05T09:34:54.874Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"6c0fefbb-adee-4ac2-aa18-c9dc25e78a85","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-06-05T09:34:54.887Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"6c0fefbb-adee-4ac2-aa18-c9dc25e78a85","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-06-05T09:34:54.887Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"6c0fefbb-adee-4ac2-aa18-c9dc25e78a85","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:34:56.862Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"10331632-9684-4dc6-ba9d-c9bee6196ba2"}
{"level":"info","ts":"2026-06-05T09:34:56.862Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"10331632-9684-4dc6-ba9d-c9bee6196ba2"}
{"level":"info","ts":"2026-06-05T09:34:56.862Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"10331632-9684-4dc6-ba9d-c9bee6196ba2","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-05T09:34:56.862Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"10331632-9684-4dc6-ba9d-c9bee6196ba2","Found 1 head Pod":"raycluster-auth-head-k2fq2","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:34:56.862Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"10331632-9684-4dc6-ba9d-c9bee6196ba2","head Pod":"raycluster-auth-head-k2fq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-k2fq2. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:34:56.862Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"10331632-9684-4dc6-ba9d-c9bee6196ba2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:34:56.862Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"10331632-9684-4dc6-ba9d-c9bee6196ba2","worker Pod":"raycluster-auth-small-group-worker-w5wpq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-w5wpq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:34:56.862Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"10331632-9684-4dc6-ba9d-c9bee6196ba2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:34:56.862Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"10331632-9684-4dc6-ba9d-c9bee6196ba2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:34:56.862Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"10331632-9684-4dc6-ba9d-c9bee6196ba2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:34:56.862Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"10331632-9684-4dc6-ba9d-c9bee6196ba2","podName":"raycluster-auth-head-k2fq2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:34:56.862Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"10331632-9684-4dc6-ba9d-c9bee6196ba2","podName":"raycluster-auth-head-k2fq2","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:34:56.863Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"10331632-9684-4dc6-ba9d-c9bee6196ba2","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:34:56.863Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"10331632-9684-4dc6-ba9d-c9bee6196ba2","seconds":300}
{"level":"info","ts":"2026-06-05T09:34:57.902Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0c2d5e63-e33d-4d07-bf7f-f236e66a1343"}
{"level":"info","ts":"2026-06-05T09:34:57.902Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0c2d5e63-e33d-4d07-bf7f-f236e66a1343"}
{"level":"info","ts":"2026-06-05T09:34:57.902Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0c2d5e63-e33d-4d07-bf7f-f236e66a1343","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-05T09:34:57.902Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0c2d5e63-e33d-4d07-bf7f-f236e66a1343","Found 1 head Pod":"raycluster-auth-head-k2fq2","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:34:57.902Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0c2d5e63-e33d-4d07-bf7f-f236e66a1343","head Pod":"raycluster-auth-head-k2fq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-k2fq2. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:34:57.902Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0c2d5e63-e33d-4d07-bf7f-f236e66a1343","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:34:57.902Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0c2d5e63-e33d-4d07-bf7f-f236e66a1343","worker Pod":"raycluster-auth-small-group-worker-w5wpq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-w5wpq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:34:57.902Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0c2d5e63-e33d-4d07-bf7f-f236e66a1343","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:34:57.902Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0c2d5e63-e33d-4d07-bf7f-f236e66a1343","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:34:57.902Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0c2d5e63-e33d-4d07-bf7f-f236e66a1343","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:34:57.902Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0c2d5e63-e33d-4d07-bf7f-f236e66a1343","podName":"raycluster-auth-head-k2fq2","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:30:43Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:34:57.902Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0c2d5e63-e33d-4d07-bf7f-f236e66a1343","podName":"raycluster-auth-head-k2fq2","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:30:43Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:34:57.902Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0c2d5e63-e33d-4d07-bf7f-f236e66a1343","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:30:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:30:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-05T09:34:57.902Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"0c2d5e63-e33d-4d07-bf7f-f236e66a1343","name":"raycluster-auth","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:34:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.14","serviceIP":"10.132.0.14","podName":"raycluster-auth-head-k2fq2","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:30:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:34:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.14","serviceIP":"10.132.0.14","podName":"raycluster-auth-head-k2fq2","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:30:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:34:57.922Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"4be3da74-5f02-46eb-9a0d-95c6377a59e2","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:34:57.922Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"c8bcda4a-983d-4af2-97f7-301ea34caf4d","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:34:57.922Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"4be3da74-5f02-46eb-9a0d-95c6377a59e2","pod":"raycluster-auth-small-group-worker-w5wpq","ip":"10.133.0.34"}
{"level":"info","ts":"2026-06-05T09:34:57.922Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"4be3da74-5f02-46eb-9a0d-95c6377a59e2","pod":"raycluster-auth-head-k2fq2","ip":"10.132.0.14"}
{"level":"info","ts":"2026-06-05T09:34:57.922Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"4be3da74-5f02-46eb-9a0d-95c6377a59e2","rayCluster":"raycluster-auth","podIPs":["10.133.0.34","10.132.0.14"]}
{"level":"info","ts":"2026-06-05T09:34:57.933Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"c8bcda4a-983d-4af2-97f7-301ea34caf4d","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-06-05T09:34:57.946Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"c8bcda4a-983d-4af2-97f7-301ea34caf4d","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-06-05T09:34:57.946Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"c8bcda4a-983d-4af2-97f7-301ea34caf4d","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:35:00.006Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"325a0dc0-ccf9-4622-9842-80e723db71d1"}
{"level":"info","ts":"2026-06-05T09:35:00.006Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"325a0dc0-ccf9-4622-9842-80e723db71d1"}
{"level":"info","ts":"2026-06-05T09:35:00.006Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"325a0dc0-ccf9-4622-9842-80e723db71d1","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-05T09:35:00.006Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"325a0dc0-ccf9-4622-9842-80e723db71d1","Found 1 head Pod":"raycluster-auth-head-k2fq2","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:35:00.006Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"325a0dc0-ccf9-4622-9842-80e723db71d1","head Pod":"raycluster-auth-head-k2fq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-k2fq2. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:35:00.006Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"325a0dc0-ccf9-4622-9842-80e723db71d1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:35:00.006Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"325a0dc0-ccf9-4622-9842-80e723db71d1","worker Pod":"raycluster-auth-small-group-worker-w5wpq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-w5wpq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:35:00.006Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"325a0dc0-ccf9-4622-9842-80e723db71d1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:35:00.006Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"325a0dc0-ccf9-4622-9842-80e723db71d1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:35:00.006Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"325a0dc0-ccf9-4622-9842-80e723db71d1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:35:00.006Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"325a0dc0-ccf9-4622-9842-80e723db71d1","podName":"raycluster-auth-head-k2fq2","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:30:43Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:35:00.006Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"325a0dc0-ccf9-4622-9842-80e723db71d1","podName":"raycluster-auth-head-k2fq2","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:30:43Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:35:00.006Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"325a0dc0-ccf9-4622-9842-80e723db71d1","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:35:00.006Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"325a0dc0-ccf9-4622-9842-80e723db71d1","seconds":300}
{"level":"info","ts":"2026-06-05T09:35:03.930Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"394fc44b-13aa-434e-9567-06cd4c842080"}
{"level":"info","ts":"2026-06-05T09:35:03.930Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"394fc44b-13aa-434e-9567-06cd4c842080"}
{"level":"info","ts":"2026-06-05T09:35:03.930Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"394fc44b-13aa-434e-9567-06cd4c842080","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-05T09:35:03.930Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"394fc44b-13aa-434e-9567-06cd4c842080","Found 1 head Pod":"raycluster-auth-head-k2fq2","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:35:03.930Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"394fc44b-13aa-434e-9567-06cd4c842080","head Pod":"raycluster-auth-head-k2fq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-k2fq2. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:35:03.930Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"394fc44b-13aa-434e-9567-06cd4c842080","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:35:03.930Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"394fc44b-13aa-434e-9567-06cd4c842080","worker Pod":"raycluster-auth-small-group-worker-w5wpq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-w5wpq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:35:03.930Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"394fc44b-13aa-434e-9567-06cd4c842080","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:35:03.930Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"394fc44b-13aa-434e-9567-06cd4c842080","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:35:03.930Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"394fc44b-13aa-434e-9567-06cd4c842080","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:35:03.930Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"394fc44b-13aa-434e-9567-06cd4c842080","podName":"raycluster-auth-small-group-worker-w5wpq","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:30:43Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-05T09:35:03.930Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"394fc44b-13aa-434e-9567-06cd4c842080","podName":"raycluster-auth-small-group-worker-w5wpq","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:30:43Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-05T09:35:03.930Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"394fc44b-13aa-434e-9567-06cd4c842080","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:30:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:35:03Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:30:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-05T09:35:03.930Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"394fc44b-13aa-434e-9567-06cd4c842080","name":"raycluster-auth","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:34:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.14","serviceIP":"10.132.0.14","podName":"raycluster-auth-head-k2fq2","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:30:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:35:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.14","serviceIP":"10.132.0.14","podName":"raycluster-auth-head-k2fq2","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:35:03Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:30:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:35:03.943Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"8eea9ef3-832f-4df8-acb1-e87a77b781b1","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:35:03.943Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"67f2453e-3d78-4e03-8611-25f9c4689ae4","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:35:03.943Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"67f2453e-3d78-4e03-8611-25f9c4689ae4","pod":"raycluster-auth-head-k2fq2","ip":"10.132.0.14"}
{"level":"info","ts":"2026-06-05T09:35:03.943Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"67f2453e-3d78-4e03-8611-25f9c4689ae4","pod":"raycluster-auth-small-group-worker-w5wpq","ip":"10.133.0.34"}
{"level":"info","ts":"2026-06-05T09:35:03.943Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"67f2453e-3d78-4e03-8611-25f9c4689ae4","rayCluster":"raycluster-auth","podIPs":["10.132.0.14","10.133.0.34"]}
{"level":"info","ts":"2026-06-05T09:35:03.956Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"8eea9ef3-832f-4df8-acb1-e87a77b781b1","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-06-05T09:35:03.966Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"8eea9ef3-832f-4df8-acb1-e87a77b781b1","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-06-05T09:35:03.967Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"8eea9ef3-832f-4df8-acb1-e87a77b781b1","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:35:05.874Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"9ee6057f-a358-43d5-a7d2-8108615e3d54"}
{"level":"info","ts":"2026-06-05T09:35:05.874Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"9ee6057f-a358-43d5-a7d2-8108615e3d54"}
{"level":"info","ts":"2026-06-05T09:35:05.874Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"9ee6057f-a358-43d5-a7d2-8108615e3d54","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-05T09:35:05.874Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"9ee6057f-a358-43d5-a7d2-8108615e3d54","Found 1 head Pod":"raycluster-auth-head-k2fq2","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:35:05.874Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"9ee6057f-a358-43d5-a7d2-8108615e3d54","head Pod":"raycluster-auth-head-k2fq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-k2fq2. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:35:05.874Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"9ee6057f-a358-43d5-a7d2-8108615e3d54","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:35:05.874Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"9ee6057f-a358-43d5-a7d2-8108615e3d54","worker Pod":"raycluster-auth-small-group-worker-w5wpq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-w5wpq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:35:05.874Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"9ee6057f-a358-43d5-a7d2-8108615e3d54","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:35:05.874Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"9ee6057f-a358-43d5-a7d2-8108615e3d54","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:35:05.874Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"9ee6057f-a358-43d5-a7d2-8108615e3d54","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:35:05.875Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"9ee6057f-a358-43d5-a7d2-8108615e3d54","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-06-05T09:35:05.875Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"9ee6057f-a358-43d5-a7d2-8108615e3d54","name":"raycluster-auth","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:35:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.14","serviceIP":"10.132.0.14","podName":"raycluster-auth-head-k2fq2","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:35:03Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:30:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:35:05Z","stateTransitionTimes":{"ready":"2026-06-05T09:35:05Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.14","serviceIP":"10.132.0.14","podName":"raycluster-auth-head-k2fq2","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:35:03Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:35:05Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:30:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:35:05.888Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"6164430f-a66a-49fa-930a-8db748cdfe2b","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:35:05.888Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"b2a4470f-b69a-45b9-8856-014e52559019","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:35:05.888Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"6164430f-a66a-49fa-930a-8db748cdfe2b","pod":"raycluster-auth-head-k2fq2","ip":"10.132.0.14"}
{"level":"info","ts":"2026-06-05T09:35:05.888Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"6164430f-a66a-49fa-930a-8db748cdfe2b","pod":"raycluster-auth-small-group-worker-w5wpq","ip":"10.133.0.34"}
{"level":"info","ts":"2026-06-05T09:35:05.888Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"6164430f-a66a-49fa-930a-8db748cdfe2b","rayCluster":"raycluster-auth","podIPs":["10.132.0.14","10.133.0.34"]}
{"level":"info","ts":"2026-06-05T09:35:05.903Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"b2a4470f-b69a-45b9-8856-014e52559019","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-06-05T09:35:05.914Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"b2a4470f-b69a-45b9-8856-014e52559019","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-06-05T09:35:05.914Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"b2a4470f-b69a-45b9-8856-014e52559019","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:35:05.944Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"ce9636ab-ca15-452d-8246-12937fdfeedc"}
{"level":"info","ts":"2026-06-05T09:35:05.944Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"ce9636ab-ca15-452d-8246-12937fdfeedc"}
{"level":"info","ts":"2026-06-05T09:35:05.944Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"ce9636ab-ca15-452d-8246-12937fdfeedc","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-05T09:35:05.944Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"ce9636ab-ca15-452d-8246-12937fdfeedc","Found 1 head Pod":"raycluster-auth-head-k2fq2","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:35:05.944Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"ce9636ab-ca15-452d-8246-12937fdfeedc","head Pod":"raycluster-auth-head-k2fq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-k2fq2. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:35:05.944Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"ce9636ab-ca15-452d-8246-12937fdfeedc","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:35:05.944Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"ce9636ab-ca15-452d-8246-12937fdfeedc","worker Pod":"raycluster-auth-small-group-worker-w5wpq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-w5wpq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:35:05.944Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"ce9636ab-ca15-452d-8246-12937fdfeedc","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:35:05.944Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"ce9636ab-ca15-452d-8246-12937fdfeedc","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:35:05.944Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"ce9636ab-ca15-452d-8246-12937fdfeedc","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:35:05.944Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"ce9636ab-ca15-452d-8246-12937fdfeedc","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:35:05.944Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"ce9636ab-ca15-452d-8246-12937fdfeedc","seconds":300}
{"level":"info","ts":"2026-06-05T09:35:06.325Z","logger":"raycluster-resource","msg":"default","name":"raycluster"}
{"level":"info","ts":"2026-06-05T09:35:06.325Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"raycluster","namespace":"test-ns-rfwvh"}
{"level":"info","ts":"2026-06-05T09:35:06.339Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"6e4f7d95-3aab-4fb6-944c-2054ee16a92c","namespacedName":{"name":"raycluster","namespace":"test-ns-rfwvh"}}
{"level":"info","ts":"2026-06-05T09:35:06.340Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"6e4f7d95-3aab-4fb6-944c-2054ee16a92c","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:35:06.340Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"6e4f7d95-3aab-4fb6-944c-2054ee16a92c","cluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:35:06.340Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"6e4f7d95-3aab-4fb6-944c-2054ee16a92c","cluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:35:06.339Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"00a1dacf-46d2-4af8-bc12-3955a800b09d","cluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:35:06.339Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"d44ec7f7-a17d-47ac-9279-e244dc7d696c"}
{"level":"info","ts":"2026-06-05T09:35:06.340Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"d44ec7f7-a17d-47ac-9279-e244dc7d696c"}
{"level":"info","ts":"2026-06-05T09:35:06.339Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"a48dcb28-4cbe-438c-920d-0609fcfefef7","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:35:06.340Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"a48dcb28-4cbe-438c-920d-0609fcfefef7","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:35:06.346Z","logger":"raycluster-resource","msg":"default","name":"raycluster"}
{"level":"info","ts":"2026-06-05T09:35:06.346Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"raycluster","namespace":"test-ns-rfwvh"}
{"level":"info","ts":"2026-06-05T09:35:06.348Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"d44ec7f7-a17d-47ac-9279-e244dc7d696c","name":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-05T09:35:06.348Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"d44ec7f7-a17d-47ac-9279-e244dc7d696c","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-06-05T09:35:06.348Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"d44ec7f7-a17d-47ac-9279-e244dc7d696c","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:35:06.348Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"d44ec7f7-a17d-47ac-9279-e244dc7d696c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-06-05T09:35:06.348Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"d44ec7f7-a17d-47ac-9279-e244dc7d696c","filter labels":["test-ns-rfwvh",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:35:06.348Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"d44ec7f7-a17d-47ac-9279-e244dc7d696c","filter labels":["test-ns-rfwvh",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:35:06.348Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"d44ec7f7-a17d-47ac-9279-e244dc7d696c","filter labels":["test-ns-rfwvh",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:35:06.348Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"d44ec7f7-a17d-47ac-9279-e244dc7d696c","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-05T09:35:06.348Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"d44ec7f7-a17d-47ac-9279-e244dc7d696c","name":"raycluster","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:35:06Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:35:06.350Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"a48dcb28-4cbe-438c-920d-0609fcfefef7","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:35:06.350Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"a48dcb28-4cbe-438c-920d-0609fcfefef7","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:35:06.359Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"00a1dacf-46d2-4af8-bc12-3955a800b09d","cluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:35:06.359Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"d49c9806-5a8b-47bf-b0d9-05ca08999e9b","cluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:35:06.406Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"a48dcb28-4cbe-438c-920d-0609fcfefef7"}
{"level":"info","ts":"2026-06-05T09:35:06.406Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"6e4f7d95-3aab-4fb6-944c-2054ee16a92c","name":"raycluster-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:35:06.406Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"d44ec7f7-a17d-47ac-9279-e244dc7d696c","name":"raycluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-rfwvh","name":"raycluster"}}
{"level":"info","ts":"2026-06-05T09:35:06.406Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"d44ec7f7-a17d-47ac-9279-e244dc7d696c"}
{"level":"error","ts":"2026-06-05T09:35:06.406Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"d44ec7f7-a17d-47ac-9279-e244dc7d696c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:35:06.406Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"fbc5d8f1-6a65-4de5-92f2-d1be6f9c5e04"}
{"level":"info","ts":"2026-06-05T09:35:06.406Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"fbc5d8f1-6a65-4de5-92f2-d1be6f9c5e04"}
{"level":"info","ts":"2026-06-05T09:35:06.406Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"fbc5d8f1-6a65-4de5-92f2-d1be6f9c5e04","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-05T09:35:06.406Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"fbc5d8f1-6a65-4de5-92f2-d1be6f9c5e04","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-06-05T09:35:06.406Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"fbc5d8f1-6a65-4de5-92f2-d1be6f9c5e04","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:35:06.406Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"fbc5d8f1-6a65-4de5-92f2-d1be6f9c5e04","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-06-05T09:35:06.406Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"fbc5d8f1-6a65-4de5-92f2-d1be6f9c5e04","filter labels":["test-ns-rfwvh",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:35:06.406Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"a48dcb28-4cbe-438c-920d-0609fcfefef7","rayCluster":"raycluster","secret":"ca-secret"}
{"level":"info","ts":"2026-06-05T09:35:06.406Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"a48dcb28-4cbe-438c-920d-0609fcfefef7","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:35:06.406Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"fbc5d8f1-6a65-4de5-92f2-d1be6f9c5e04","filter labels":["test-ns-rfwvh",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:35:06.406Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"fbc5d8f1-6a65-4de5-92f2-d1be6f9c5e04","filter labels":["test-ns-rfwvh",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:35:06.406Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"fbc5d8f1-6a65-4de5-92f2-d1be6f9c5e04","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-05T09:35:06.406Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"fbc5d8f1-6a65-4de5-92f2-d1be6f9c5e04","name":"raycluster","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:35:06Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:35:06.412Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"d49c9806-5a8b-47bf-b0d9-05ca08999e9b","name":"raycluster-head"}
{"level":"info","ts":"2026-06-05T09:35:06.414Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"6e4f7d95-3aab-4fb6-944c-2054ee16a92c","name":"kuberay-gateway-access","namespace":"test-ns-rfwvh","operation":"created"}
{"level":"info","ts":"2026-06-05T09:35:06.424Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"fbc5d8f1-6a65-4de5-92f2-d1be6f9c5e04"}
{"level":"error","ts":"2026-06-05T09:35:06.424Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"fbc5d8f1-6a65-4de5-92f2-d1be6f9c5e04","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:35:06.424Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"39ca3e10-40eb-4ffe-b78c-f7a89d4e67b0"}
{"level":"info","ts":"2026-06-05T09:35:06.424Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"39ca3e10-40eb-4ffe-b78c-f7a89d4e67b0"}
{"level":"info","ts":"2026-06-05T09:35:06.424Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"39ca3e10-40eb-4ffe-b78c-f7a89d4e67b0","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-05T09:35:06.424Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"39ca3e10-40eb-4ffe-b78c-f7a89d4e67b0","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-06-05T09:35:06.424Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"39ca3e10-40eb-4ffe-b78c-f7a89d4e67b0","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:35:06.424Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"39ca3e10-40eb-4ffe-b78c-f7a89d4e67b0","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-06-05T09:35:06.424Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"39ca3e10-40eb-4ffe-b78c-f7a89d4e67b0","filter labels":["test-ns-rfwvh",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:35:06.424Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"39ca3e10-40eb-4ffe-b78c-f7a89d4e67b0","filter labels":["test-ns-rfwvh",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:35:06.424Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"39ca3e10-40eb-4ffe-b78c-f7a89d4e67b0","filter labels":["test-ns-rfwvh",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:35:06.424Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"39ca3e10-40eb-4ffe-b78c-f7a89d4e67b0"}
{"level":"error","ts":"2026-06-05T09:35:06.424Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"39ca3e10-40eb-4ffe-b78c-f7a89d4e67b0","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:35:06.425Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"a48dcb28-4cbe-438c-920d-0609fcfefef7","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:35:06.505Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"b4d9825f-3a23-4c6b-9895-37037076d5d3"}
{"level":"info","ts":"2026-06-05T09:35:06.506Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"b4d9825f-3a23-4c6b-9895-37037076d5d3"}
{"level":"info","ts":"2026-06-05T09:35:06.506Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"b4d9825f-3a23-4c6b-9895-37037076d5d3","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-05T09:35:06.506Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"b4d9825f-3a23-4c6b-9895-37037076d5d3","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-06-05T09:35:06.506Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"a48dcb28-4cbe-438c-920d-0609fcfefef7"}
{"level":"info","ts":"2026-06-05T09:35:06.506Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"a48dcb28-4cbe-438c-920d-0609fcfefef7","rayCluster":"raycluster","podIPs":[]}
{"level":"error","ts":"2026-06-05T09:35:06.506Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"b4d9825f-3a23-4c6b-9895-37037076d5d3","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:35:06.506Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"b4d9825f-3a23-4c6b-9895-37037076d5d3","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-06-05T09:35:06.506Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"b4d9825f-3a23-4c6b-9895-37037076d5d3","filter labels":["test-ns-rfwvh",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:35:06.506Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"d49c9806-5a8b-47bf-b0d9-05ca08999e9b","name":"raycluster-workers"}
{"level":"info","ts":"2026-06-05T09:35:06.506Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"b4d9825f-3a23-4c6b-9895-37037076d5d3","filter labels":["test-ns-rfwvh",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:35:06.506Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"b4d9825f-3a23-4c6b-9895-37037076d5d3","filter labels":["test-ns-rfwvh",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:35:06.506Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"6e4f7d95-3aab-4fb6-944c-2054ee16a92c","name":"test-ns-rfwvh-raycluster","operation":"created"}
{"level":"info","ts":"2026-06-05T09:35:06.506Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"b4d9825f-3a23-4c6b-9895-37037076d5d3"}
{"level":"info","ts":"2026-06-05T09:35:06.506Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"d49c9806-5a8b-47bf-b0d9-05ca08999e9b","cluster":"raycluster"}
{"level":"error","ts":"2026-06-05T09:35:06.506Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"b4d9825f-3a23-4c6b-9895-37037076d5d3","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:35:06.506Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"a4248301-b95f-423a-a765-4b9f309de701","cluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:35:06.516Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"6e4f7d95-3aab-4fb6-944c-2054ee16a92c","name":"kube-rbac-proxy-config-raycluster","operation":"created"}
{"level":"info","ts":"2026-06-05T09:35:06.516Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"a48dcb28-4cbe-438c-920d-0609fcfefef7"}
{"level":"info","ts":"2026-06-05T09:35:06.517Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"a48dcb28-4cbe-438c-920d-0609fcfefef7","rayCluster":"raycluster","podIPs":[]}
{"level":"info","ts":"2026-06-05T09:35:06.517Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"a48dcb28-4cbe-438c-920d-0609fcfefef7"}
{"level":"info","ts":"2026-06-05T09:35:06.517Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"4ac8ea43-fa30-42d4-9bb6-5b3e9193c533","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:35:06.517Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"4ac8ea43-fa30-42d4-9bb6-5b3e9193c533"}
{"level":"info","ts":"2026-06-05T09:35:06.522Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"a4248301-b95f-423a-a765-4b9f309de701","name":"raycluster-head"}
{"level":"error","ts":"2026-06-05T09:35:06.523Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"6e4f7d95-3aab-4fb6-944c-2054ee16a92c","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"raycluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:35:06.523Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"6e4f7d95-3aab-4fb6-944c-2054ee16a92c"}
{"level":"error","ts":"2026-06-05T09:35:06.523Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"6e4f7d95-3aab-4fb6-944c-2054ee16a92c","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"raycluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:35:06.523Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"bf5159e6-5418-4265-9776-6d3ba4185fb0","namespacedName":{"name":"raycluster","namespace":"test-ns-rfwvh"}}
{"level":"info","ts":"2026-06-05T09:35:06.523Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"bf5159e6-5418-4265-9776-6d3ba4185fb0","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:35:06.523Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"bf5159e6-5418-4265-9776-6d3ba4185fb0","cluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:35:06.536Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"bf5159e6-5418-4265-9776-6d3ba4185fb0","cluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:35:06.536Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"2eb99609-3c46-4fd8-9f8c-78225970a92c","namespacedName":{"name":"raycluster","namespace":"test-ns-rfwvh"}}
{"level":"info","ts":"2026-06-05T09:35:06.536Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"2eb99609-3c46-4fd8-9f8c-78225970a92c","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:35:06.536Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"2eb99609-3c46-4fd8-9f8c-78225970a92c","cluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:35:06.543Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"a4248301-b95f-423a-a765-4b9f309de701","name":"raycluster-workers"}
{"level":"info","ts":"2026-06-05T09:35:06.543Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"a4248301-b95f-423a-a765-4b9f309de701","cluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:35:06.544Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"01ffa7ef-ccdd-40a4-a212-bc7c4fbd11fc","cluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:35:06.605Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"1f393dbc-6aad-4ff0-b416-f2281c54a10c","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:35:06.605Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"20c9bd93-7d06-4d93-8a3f-72c229116a46"}
{"level":"info","ts":"2026-06-05T09:35:06.605Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"20c9bd93-7d06-4d93-8a3f-72c229116a46"}
{"level":"info","ts":"2026-06-05T09:35:06.606Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"20c9bd93-7d06-4d93-8a3f-72c229116a46","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-05T09:35:06.606Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"20c9bd93-7d06-4d93-8a3f-72c229116a46","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-06-05T09:35:06.606Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"1f393dbc-6aad-4ff0-b416-f2281c54a10c"}
{"level":"error","ts":"2026-06-05T09:35:06.606Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"20c9bd93-7d06-4d93-8a3f-72c229116a46","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:35:06.606Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"20c9bd93-7d06-4d93-8a3f-72c229116a46","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-06-05T09:35:06.606Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"2eb99609-3c46-4fd8-9f8c-78225970a92c","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"raycluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:35:06.606Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"20c9bd93-7d06-4d93-8a3f-72c229116a46","filter labels":["test-ns-rfwvh",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:35:06.606Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"2eb99609-3c46-4fd8-9f8c-78225970a92c"}
{"level":"error","ts":"2026-06-05T09:35:06.606Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"2eb99609-3c46-4fd8-9f8c-78225970a92c","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"raycluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:35:06.606Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"20c9bd93-7d06-4d93-8a3f-72c229116a46","filter labels":["test-ns-rfwvh",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:35:06.606Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"20c9bd93-7d06-4d93-8a3f-72c229116a46","filter labels":["test-ns-rfwvh",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:35:06.606Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"20c9bd93-7d06-4d93-8a3f-72c229116a46"}
{"level":"error","ts":"2026-06-05T09:35:06.606Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"20c9bd93-7d06-4d93-8a3f-72c229116a46","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:35:06.611Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"08bb3365-3d04-4624-9b13-658975b8d9a8","namespacedName":{"name":"raycluster","namespace":"test-ns-rfwvh"}}
{"level":"info","ts":"2026-06-05T09:35:06.611Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"08bb3365-3d04-4624-9b13-658975b8d9a8","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:35:06.611Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"08bb3365-3d04-4624-9b13-658975b8d9a8","cluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:35:06.611Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"01ffa7ef-ccdd-40a4-a212-bc7c4fbd11fc","name":"raycluster-head"}
{"level":"info","ts":"2026-06-05T09:35:06.621Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"08bb3365-3d04-4624-9b13-658975b8d9a8","cluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:35:06.622Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"01ffa7ef-ccdd-40a4-a212-bc7c4fbd11fc","name":"raycluster-workers"}
{"level":"info","ts":"2026-06-05T09:35:06.622Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"01ffa7ef-ccdd-40a4-a212-bc7c4fbd11fc","cluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:35:06.687Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"e2aebe0c-6eaa-4dd9-bd28-356418fac8ed"}
{"level":"info","ts":"2026-06-05T09:35:06.687Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"e2aebe0c-6eaa-4dd9-bd28-356418fac8ed"}
{"level":"info","ts":"2026-06-05T09:35:06.687Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"e2aebe0c-6eaa-4dd9-bd28-356418fac8ed","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-05T09:35:06.687Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"e2aebe0c-6eaa-4dd9-bd28-356418fac8ed","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-06-05T09:35:06.687Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"e2aebe0c-6eaa-4dd9-bd28-356418fac8ed","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:35:06.687Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"e2aebe0c-6eaa-4dd9-bd28-356418fac8ed","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-06-05T09:35:06.687Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"e2aebe0c-6eaa-4dd9-bd28-356418fac8ed","filter labels":["test-ns-rfwvh",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:35:06.687Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"e2aebe0c-6eaa-4dd9-bd28-356418fac8ed","filter labels":["test-ns-rfwvh",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:35:06.687Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"e2aebe0c-6eaa-4dd9-bd28-356418fac8ed","filter labels":["test-ns-rfwvh",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:35:06.687Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"e2aebe0c-6eaa-4dd9-bd28-356418fac8ed"}
{"level":"error","ts":"2026-06-05T09:35:06.687Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"e2aebe0c-6eaa-4dd9-bd28-356418fac8ed","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:35:06.848Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"ed07f181-220e-4045-97b0-eff0e1136bdf"}
{"level":"info","ts":"2026-06-05T09:35:06.848Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"ed07f181-220e-4045-97b0-eff0e1136bdf"}
{"level":"info","ts":"2026-06-05T09:35:06.848Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"ed07f181-220e-4045-97b0-eff0e1136bdf","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-05T09:35:06.848Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"ed07f181-220e-4045-97b0-eff0e1136bdf","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-06-05T09:35:06.848Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"ed07f181-220e-4045-97b0-eff0e1136bdf","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:35:06.848Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"ed07f181-220e-4045-97b0-eff0e1136bdf","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-06-05T09:35:06.848Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"ed07f181-220e-4045-97b0-eff0e1136bdf","filter labels":["test-ns-rfwvh",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:35:06.848Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"ed07f181-220e-4045-97b0-eff0e1136bdf","filter labels":["test-ns-rfwvh",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:35:06.848Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"ed07f181-220e-4045-97b0-eff0e1136bdf","filter labels":["test-ns-rfwvh",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:35:06.848Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"ed07f181-220e-4045-97b0-eff0e1136bdf"}
{"level":"error","ts":"2026-06-05T09:35:06.848Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"ed07f181-220e-4045-97b0-eff0e1136bdf","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:35:07.168Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"9fdbc8b2-3ccd-45a9-bcbf-132b5f813234"}
{"level":"info","ts":"2026-06-05T09:35:07.168Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"9fdbc8b2-3ccd-45a9-bcbf-132b5f813234"}
{"level":"info","ts":"2026-06-05T09:35:07.169Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"9fdbc8b2-3ccd-45a9-bcbf-132b5f813234","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-05T09:35:07.169Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"9fdbc8b2-3ccd-45a9-bcbf-132b5f813234","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-06-05T09:35:07.169Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"9fdbc8b2-3ccd-45a9-bcbf-132b5f813234","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:35:07.169Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"9fdbc8b2-3ccd-45a9-bcbf-132b5f813234","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-06-05T09:35:07.169Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"9fdbc8b2-3ccd-45a9-bcbf-132b5f813234","filter labels":["test-ns-rfwvh",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:35:07.169Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"9fdbc8b2-3ccd-45a9-bcbf-132b5f813234","filter labels":["test-ns-rfwvh",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:35:07.169Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"9fdbc8b2-3ccd-45a9-bcbf-132b5f813234","filter labels":["test-ns-rfwvh",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:35:07.169Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"9fdbc8b2-3ccd-45a9-bcbf-132b5f813234"}
{"level":"error","ts":"2026-06-05T09:35:07.169Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"9fdbc8b2-3ccd-45a9-bcbf-132b5f813234","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:35:07.809Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"4780387f-8885-4e77-8b17-5bd1dbac2345"}
{"level":"info","ts":"2026-06-05T09:35:07.809Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"4780387f-8885-4e77-8b17-5bd1dbac2345"}
{"level":"info","ts":"2026-06-05T09:35:07.810Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"4780387f-8885-4e77-8b17-5bd1dbac2345","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-05T09:35:07.810Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"4780387f-8885-4e77-8b17-5bd1dbac2345","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-06-05T09:35:07.810Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"4780387f-8885-4e77-8b17-5bd1dbac2345","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:35:07.810Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"4780387f-8885-4e77-8b17-5bd1dbac2345","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-06-05T09:35:07.810Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"4780387f-8885-4e77-8b17-5bd1dbac2345","filter labels":["test-ns-rfwvh",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:35:07.810Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"4780387f-8885-4e77-8b17-5bd1dbac2345","filter labels":["test-ns-rfwvh",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:35:07.810Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"4780387f-8885-4e77-8b17-5bd1dbac2345","filter labels":["test-ns-rfwvh",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:35:07.810Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"4780387f-8885-4e77-8b17-5bd1dbac2345"}
{"level":"error","ts":"2026-06-05T09:35:07.810Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"4780387f-8885-4e77-8b17-5bd1dbac2345","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:35:09.090Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"cd22f87a-5a9b-4f28-8ce4-d998a24354ca"}
{"level":"info","ts":"2026-06-05T09:35:09.090Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"cd22f87a-5a9b-4f28-8ce4-d998a24354ca"}
{"level":"info","ts":"2026-06-05T09:35:09.090Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"cd22f87a-5a9b-4f28-8ce4-d998a24354ca","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-05T09:35:09.091Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"cd22f87a-5a9b-4f28-8ce4-d998a24354ca"}
{"level":"info","ts":"2026-06-05T09:35:09.091Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"cd22f87a-5a9b-4f28-8ce4-d998a24354ca","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:35:09.091Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"cd22f87a-5a9b-4f28-8ce4-d998a24354ca"}
{"level":"info","ts":"2026-06-05T09:35:09.091Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"cd22f87a-5a9b-4f28-8ce4-d998a24354ca","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:35:09.091Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"cd22f87a-5a9b-4f28-8ce4-d998a24354ca","cluster":"raycluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:35:09.091Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"cd22f87a-5a9b-4f28-8ce4-d998a24354ca","cluster":"raycluster","authType":"IntegratedOAuth","serviceAccount":"raycluster-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-05T09:35:09.091Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"cd22f87a-5a9b-4f28-8ce4-d998a24354ca","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"raycluster","ray.io/group":"headgroup","ray.io/identifier":"raycluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-05T09:35:09.091Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"cd22f87a-5a9b-4f28-8ce4-d998a24354ca","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"raycluster-head-svc.test-ns-rfwvh.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-05T09:35:09.091Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"cd22f87a-5a9b-4f28-8ce4-d998a24354ca","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-head-svc.test-ns-rfwvh.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-05T09:35:09.091Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"cd22f87a-5a9b-4f28-8ce4-d998a24354ca","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-head-svc.test-ns-rfwvh.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-05T09:35:09.091Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"cd22f87a-5a9b-4f28-8ce4-d998a24354ca","enabled":true}
{"level":"info","ts":"2026-06-05T09:35:09.110Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"cd22f87a-5a9b-4f28-8ce4-d998a24354ca","name":"raycluster-head-vjd7d"}
{"level":"info","ts":"2026-06-05T09:35:09.110Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"cd22f87a-5a9b-4f28-8ce4-d998a24354ca","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:35:09.110Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"cd22f87a-5a9b-4f28-8ce4-d998a24354ca","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:35:09.110Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"cd22f87a-5a9b-4f28-8ce4-d998a24354ca","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-05T09:35:09.110Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"cd22f87a-5a9b-4f28-8ce4-d998a24354ca","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-05T09:35:09.110Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"cd22f87a-5a9b-4f28-8ce4-d998a24354ca","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-05T09:35:09.110Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"cd22f87a-5a9b-4f28-8ce4-d998a24354ca"}
{"level":"info","ts":"2026-06-05T09:35:09.110Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"cd22f87a-5a9b-4f28-8ce4-d998a24354ca","nodeType":"worker","rayStartParams":{"address":"raycluster-head-svc.test-ns-rfwvh.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-05T09:35:09.110Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"cd22f87a-5a9b-4f28-8ce4-d998a24354ca","rayStartCmd":"ray start  --address=raycluster-head-svc.test-ns-rfwvh.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:35:09.110Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"cd22f87a-5a9b-4f28-8ce4-d998a24354ca","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-head-svc.test-ns-rfwvh.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:35:09.110Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"cd22f87a-5a9b-4f28-8ce4-d998a24354ca","enabled":true}
{"level":"info","ts":"2026-06-05T09:35:09.126Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"cd22f87a-5a9b-4f28-8ce4-d998a24354ca","name":"raycluster-small-group-worker-j9pbw"}
{"level":"info","ts":"2026-06-05T09:35:09.126Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"cd22f87a-5a9b-4f28-8ce4-d998a24354ca","podName":"raycluster-head-vjd7d","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:35:09.126Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"cd22f87a-5a9b-4f28-8ce4-d998a24354ca","podName":"raycluster-head-vjd7d","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:35:09.126Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"cd22f87a-5a9b-4f28-8ce4-d998a24354ca","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"raycluster-head-svc"},"newHead":{"podName":"raycluster-head-vjd7d","serviceName":"raycluster-head-svc"}}
{"level":"info","ts":"2026-06-05T09:35:09.126Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"cd22f87a-5a9b-4f28-8ce4-d998a24354ca","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:35:06Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:35:09Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-head-vjd7d","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:35:09.138Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"f1c2639d-def5-475d-a827-3c053a127190"}
{"level":"info","ts":"2026-06-05T09:35:09.138Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"f1c2639d-def5-475d-a827-3c053a127190"}
{"level":"info","ts":"2026-06-05T09:35:09.139Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"f1c2639d-def5-475d-a827-3c053a127190","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-05T09:35:09.139Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"f1c2639d-def5-475d-a827-3c053a127190","Found 1 head Pod":"raycluster-head-vjd7d","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:35:09.139Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"f1c2639d-def5-475d-a827-3c053a127190","head Pod":"raycluster-head-vjd7d","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-vjd7d. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:35:09.139Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"f1c2639d-def5-475d-a827-3c053a127190","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:35:09.139Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"f1c2639d-def5-475d-a827-3c053a127190","worker Pod":"raycluster-small-group-worker-j9pbw","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-j9pbw. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:35:09.139Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"f1c2639d-def5-475d-a827-3c053a127190","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:35:09.139Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"f1c2639d-def5-475d-a827-3c053a127190","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:35:09.139Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"f1c2639d-def5-475d-a827-3c053a127190","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:35:09.139Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"f1c2639d-def5-475d-a827-3c053a127190","podName":"raycluster-head-vjd7d","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:35:09.139Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"e01ee0f4-6f57-4cf9-b474-4dabeee96923","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:35:09.139Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"0f740e93-f6fd-45ad-a718-b0619d54c8ab","cluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:35:09.139Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"e01ee0f4-6f57-4cf9-b474-4dabeee96923","rayCluster":"raycluster","podIPs":[]}
{"level":"info","ts":"2026-06-05T09:35:09.139Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"f1c2639d-def5-475d-a827-3c053a127190","podName":"raycluster-head-vjd7d","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:35:09.140Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"f1c2639d-def5-475d-a827-3c053a127190","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"raycluster-head-svc"},"newHead":{"podName":"raycluster-head-vjd7d","serviceName":"raycluster-head-svc"}}
{"level":"info","ts":"2026-06-05T09:35:09.140Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"f1c2639d-def5-475d-a827-3c053a127190","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:35:06Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:35:09Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-head-vjd7d","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:35:09.150Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"f1c2639d-def5-475d-a827-3c053a127190","name":"raycluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-rfwvh","name":"raycluster"}}
{"level":"info","ts":"2026-06-05T09:35:09.150Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"f1c2639d-def5-475d-a827-3c053a127190"}
{"level":"error","ts":"2026-06-05T09:35:09.150Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"f1c2639d-def5-475d-a827-3c053a127190","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:35:09.151Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"226e4a82-9b80-4090-874c-0b00d01b1b47"}
{"level":"info","ts":"2026-06-05T09:35:09.151Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"226e4a82-9b80-4090-874c-0b00d01b1b47"}
{"level":"info","ts":"2026-06-05T09:35:09.151Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"226e4a82-9b80-4090-874c-0b00d01b1b47","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-05T09:35:09.151Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"226e4a82-9b80-4090-874c-0b00d01b1b47","Found 1 head Pod":"raycluster-head-vjd7d","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:35:09.151Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"226e4a82-9b80-4090-874c-0b00d01b1b47","head Pod":"raycluster-head-vjd7d","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-vjd7d. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:35:09.151Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"226e4a82-9b80-4090-874c-0b00d01b1b47","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:35:09.151Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"226e4a82-9b80-4090-874c-0b00d01b1b47","worker Pod":"raycluster-small-group-worker-j9pbw","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-j9pbw. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:35:09.151Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"226e4a82-9b80-4090-874c-0b00d01b1b47","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:35:09.151Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"226e4a82-9b80-4090-874c-0b00d01b1b47","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:35:09.151Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"226e4a82-9b80-4090-874c-0b00d01b1b47","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:35:09.151Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"226e4a82-9b80-4090-874c-0b00d01b1b47","podName":"raycluster-head-vjd7d","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:35:09.151Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"226e4a82-9b80-4090-874c-0b00d01b1b47","podName":"raycluster-head-vjd7d","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:35:09.151Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"226e4a82-9b80-4090-874c-0b00d01b1b47","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:35:09.151Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"226e4a82-9b80-4090-874c-0b00d01b1b47","seconds":300}
{"level":"info","ts":"2026-06-05T09:35:09.153Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"1e148633-3bb3-40f5-80ac-83bf87d58c3a"}
{"level":"info","ts":"2026-06-05T09:35:09.153Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"1e148633-3bb3-40f5-80ac-83bf87d58c3a"}
{"level":"info","ts":"2026-06-05T09:35:09.156Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"0f740e93-f6fd-45ad-a718-b0619d54c8ab","name":"raycluster-head"}
{"level":"info","ts":"2026-06-05T09:35:09.167Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"0f740e93-f6fd-45ad-a718-b0619d54c8ab","name":"raycluster-workers"}
{"level":"info","ts":"2026-06-05T09:35:09.167Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"0f740e93-f6fd-45ad-a718-b0619d54c8ab","cluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:35:09.205Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"1e148633-3bb3-40f5-80ac-83bf87d58c3a","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-05T09:35:09.206Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"1e148633-3bb3-40f5-80ac-83bf87d58c3a","Found 1 head Pod":"raycluster-head-vjd7d","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:35:09.206Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"1e148633-3bb3-40f5-80ac-83bf87d58c3a","head Pod":"raycluster-head-vjd7d","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-vjd7d. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:35:09.206Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"1e148633-3bb3-40f5-80ac-83bf87d58c3a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:35:09.206Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"1e148633-3bb3-40f5-80ac-83bf87d58c3a","worker Pod":"raycluster-small-group-worker-j9pbw","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-j9pbw. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:35:09.206Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"1e148633-3bb3-40f5-80ac-83bf87d58c3a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:35:09.206Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"1e148633-3bb3-40f5-80ac-83bf87d58c3a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:35:09.206Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"1e148633-3bb3-40f5-80ac-83bf87d58c3a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:35:09.206Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"1e148633-3bb3-40f5-80ac-83bf87d58c3a","podName":"raycluster-head-vjd7d","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:35:09.206Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"1e148633-3bb3-40f5-80ac-83bf87d58c3a","podName":"raycluster-head-vjd7d","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:35:09.206Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"1e148633-3bb3-40f5-80ac-83bf87d58c3a","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-05T09:35:09.206Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"1e148633-3bb3-40f5-80ac-83bf87d58c3a","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:35:09Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-head-vjd7d","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:35:09Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-head-vjd7d","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:35:09.219Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"587e3391-c7a9-45f4-b5bd-2ad615e7d967"}
{"level":"info","ts":"2026-06-05T09:35:09.219Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"587e3391-c7a9-45f4-b5bd-2ad615e7d967"}
{"level":"info","ts":"2026-06-05T09:35:09.219Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"587e3391-c7a9-45f4-b5bd-2ad615e7d967","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-05T09:35:09.219Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"2d6f06f8-e62e-43a6-8c2f-f6fa319a8ca0","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:35:09.219Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"84d4cb4e-6814-4e73-aa4b-f78a6e287093","cluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:35:09.220Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"2d6f06f8-e62e-43a6-8c2f-f6fa319a8ca0","rayCluster":"raycluster","podIPs":[]}
{"level":"info","ts":"2026-06-05T09:35:09.219Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"587e3391-c7a9-45f4-b5bd-2ad615e7d967","Found 1 head Pod":"raycluster-head-vjd7d","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:35:09.220Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"587e3391-c7a9-45f4-b5bd-2ad615e7d967","head Pod":"raycluster-head-vjd7d","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-vjd7d. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:35:09.220Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"587e3391-c7a9-45f4-b5bd-2ad615e7d967","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:35:09.220Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"587e3391-c7a9-45f4-b5bd-2ad615e7d967","worker Pod":"raycluster-small-group-worker-j9pbw","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-j9pbw. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:35:09.220Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"587e3391-c7a9-45f4-b5bd-2ad615e7d967","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:35:09.220Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"587e3391-c7a9-45f4-b5bd-2ad615e7d967","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:35:09.220Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"587e3391-c7a9-45f4-b5bd-2ad615e7d967","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:35:09.220Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"587e3391-c7a9-45f4-b5bd-2ad615e7d967","podName":"raycluster-head-vjd7d","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:35:09.220Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"587e3391-c7a9-45f4-b5bd-2ad615e7d967","podName":"raycluster-head-vjd7d","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:35:09.220Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"587e3391-c7a9-45f4-b5bd-2ad615e7d967","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-05T09:35:09.220Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"587e3391-c7a9-45f4-b5bd-2ad615e7d967","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:35:09Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-head-vjd7d","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:35:09Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-head-vjd7d","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:35:09.227Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"587e3391-c7a9-45f4-b5bd-2ad615e7d967","name":"raycluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-rfwvh","name":"raycluster"}}
{"level":"info","ts":"2026-06-05T09:35:09.227Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"587e3391-c7a9-45f4-b5bd-2ad615e7d967"}
{"level":"error","ts":"2026-06-05T09:35:09.227Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"587e3391-c7a9-45f4-b5bd-2ad615e7d967","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:35:09.230Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"84d4cb4e-6814-4e73-aa4b-f78a6e287093","name":"raycluster-head"}
{"level":"info","ts":"2026-06-05T09:35:09.232Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"362b3749-05d7-4e11-a5da-63cf3e91caaa"}
{"level":"info","ts":"2026-06-05T09:35:09.232Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"362b3749-05d7-4e11-a5da-63cf3e91caaa"}
{"level":"info","ts":"2026-06-05T09:35:09.232Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"362b3749-05d7-4e11-a5da-63cf3e91caaa","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-05T09:35:09.233Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"362b3749-05d7-4e11-a5da-63cf3e91caaa","Found 1 head Pod":"raycluster-head-vjd7d","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:35:09.233Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"362b3749-05d7-4e11-a5da-63cf3e91caaa","head Pod":"raycluster-head-vjd7d","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-vjd7d. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:35:09.233Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"362b3749-05d7-4e11-a5da-63cf3e91caaa","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:35:09.233Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"362b3749-05d7-4e11-a5da-63cf3e91caaa","worker Pod":"raycluster-small-group-worker-j9pbw","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-j9pbw. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:35:09.233Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"362b3749-05d7-4e11-a5da-63cf3e91caaa","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:35:09.233Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"362b3749-05d7-4e11-a5da-63cf3e91caaa","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:35:09.233Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"362b3749-05d7-4e11-a5da-63cf3e91caaa","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:35:09.233Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"362b3749-05d7-4e11-a5da-63cf3e91caaa","podName":"raycluster-head-vjd7d","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:35:09.233Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"362b3749-05d7-4e11-a5da-63cf3e91caaa","podName":"raycluster-head-vjd7d","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:35:09.233Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"362b3749-05d7-4e11-a5da-63cf3e91caaa","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:35:09.233Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"362b3749-05d7-4e11-a5da-63cf3e91caaa","seconds":300}
{"level":"info","ts":"2026-06-05T09:35:09.242Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"84d4cb4e-6814-4e73-aa4b-f78a6e287093","name":"raycluster-workers"}
{"level":"info","ts":"2026-06-05T09:35:09.242Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"84d4cb4e-6814-4e73-aa4b-f78a6e287093","cluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:35:09.567Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"adcb52d3-0369-4919-93c7-79e8b925d527"}
{"level":"info","ts":"2026-06-05T09:35:09.567Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"adcb52d3-0369-4919-93c7-79e8b925d527"}
{"level":"info","ts":"2026-06-05T09:35:09.567Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"adcb52d3-0369-4919-93c7-79e8b925d527","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-05T09:35:09.567Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"adcb52d3-0369-4919-93c7-79e8b925d527","Found 1 head Pod":"raycluster-head-vjd7d","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:35:09.567Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"adcb52d3-0369-4919-93c7-79e8b925d527","head Pod":"raycluster-head-vjd7d","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-vjd7d. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:35:09.567Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"adcb52d3-0369-4919-93c7-79e8b925d527","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:35:09.567Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"adcb52d3-0369-4919-93c7-79e8b925d527","worker Pod":"raycluster-small-group-worker-j9pbw","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-j9pbw. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:35:09.567Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"adcb52d3-0369-4919-93c7-79e8b925d527","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:35:09.567Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"adcb52d3-0369-4919-93c7-79e8b925d527","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:35:09.567Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"adcb52d3-0369-4919-93c7-79e8b925d527","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:35:09.567Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"adcb52d3-0369-4919-93c7-79e8b925d527","podName":"raycluster-head-vjd7d","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:35:09.567Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"adcb52d3-0369-4919-93c7-79e8b925d527","podName":"raycluster-head-vjd7d","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:35:09.567Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"adcb52d3-0369-4919-93c7-79e8b925d527","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:35:09.567Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"adcb52d3-0369-4919-93c7-79e8b925d527","seconds":300}
{"level":"info","ts":"2026-06-05T09:35:09.590Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"6d424d7a-3adb-4a91-a43a-3e3988131e5c"}
{"level":"info","ts":"2026-06-05T09:35:09.590Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"6d424d7a-3adb-4a91-a43a-3e3988131e5c"}
{"level":"info","ts":"2026-06-05T09:35:09.590Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"6d424d7a-3adb-4a91-a43a-3e3988131e5c","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-05T09:35:09.590Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"6d424d7a-3adb-4a91-a43a-3e3988131e5c","Found 1 head Pod":"raycluster-head-vjd7d","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:35:09.590Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"6d424d7a-3adb-4a91-a43a-3e3988131e5c","head Pod":"raycluster-head-vjd7d","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-vjd7d. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:35:09.590Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"6d424d7a-3adb-4a91-a43a-3e3988131e5c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:35:09.590Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"6d424d7a-3adb-4a91-a43a-3e3988131e5c","worker Pod":"raycluster-small-group-worker-j9pbw","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-j9pbw. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:35:09.590Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"6d424d7a-3adb-4a91-a43a-3e3988131e5c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:35:09.590Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"6d424d7a-3adb-4a91-a43a-3e3988131e5c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:35:09.590Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"6d424d7a-3adb-4a91-a43a-3e3988131e5c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:35:09.590Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"6d424d7a-3adb-4a91-a43a-3e3988131e5c","podName":"raycluster-head-vjd7d","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:35:09.590Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"6d424d7a-3adb-4a91-a43a-3e3988131e5c","podName":"raycluster-head-vjd7d","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:35:09.590Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"6d424d7a-3adb-4a91-a43a-3e3988131e5c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:35:09.590Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"6d424d7a-3adb-4a91-a43a-3e3988131e5c","seconds":300}
{"level":"info","ts":"2026-06-05T09:35:09.923Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"117d0e1c-a089-4c2c-a7fd-c7211d6c5723"}
{"level":"info","ts":"2026-06-05T09:35:09.923Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"117d0e1c-a089-4c2c-a7fd-c7211d6c5723"}
{"level":"info","ts":"2026-06-05T09:35:09.923Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"117d0e1c-a089-4c2c-a7fd-c7211d6c5723","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-05T09:35:09.923Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"117d0e1c-a089-4c2c-a7fd-c7211d6c5723","Found 1 head Pod":"raycluster-head-vjd7d","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:35:09.923Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"117d0e1c-a089-4c2c-a7fd-c7211d6c5723","head Pod":"raycluster-head-vjd7d","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-vjd7d. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:35:09.923Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"117d0e1c-a089-4c2c-a7fd-c7211d6c5723","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:35:09.923Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"117d0e1c-a089-4c2c-a7fd-c7211d6c5723","worker Pod":"raycluster-small-group-worker-j9pbw","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-j9pbw. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:35:09.928Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"117d0e1c-a089-4c2c-a7fd-c7211d6c5723","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:35:09.928Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"117d0e1c-a089-4c2c-a7fd-c7211d6c5723","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:35:09.928Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"117d0e1c-a089-4c2c-a7fd-c7211d6c5723","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:35:09.929Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"117d0e1c-a089-4c2c-a7fd-c7211d6c5723","podName":"raycluster-head-vjd7d","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:35:09.929Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"117d0e1c-a089-4c2c-a7fd-c7211d6c5723","podName":"raycluster-head-vjd7d","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:35:09.929Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"117d0e1c-a089-4c2c-a7fd-c7211d6c5723","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:35:09.929Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"117d0e1c-a089-4c2c-a7fd-c7211d6c5723","seconds":300}
{"level":"info","ts":"2026-06-05T09:35:11.311Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"9173dd6a-e883-4f2e-aaae-e8645eac7d3a"}
{"level":"info","ts":"2026-06-05T09:35:11.311Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"9173dd6a-e883-4f2e-aaae-e8645eac7d3a"}
{"level":"info","ts":"2026-06-05T09:35:11.311Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"9173dd6a-e883-4f2e-aaae-e8645eac7d3a","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-05T09:35:11.311Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"9173dd6a-e883-4f2e-aaae-e8645eac7d3a","Found 1 head Pod":"raycluster-auth-head-k2fq2","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:35:11.311Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"9173dd6a-e883-4f2e-aaae-e8645eac7d3a","head Pod":"raycluster-auth-head-k2fq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-k2fq2. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:35:11.311Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"9173dd6a-e883-4f2e-aaae-e8645eac7d3a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:35:11.311Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"9173dd6a-e883-4f2e-aaae-e8645eac7d3a","worker Pod":"raycluster-auth-small-group-worker-w5wpq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-w5wpq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:35:11.311Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"9173dd6a-e883-4f2e-aaae-e8645eac7d3a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:35:11.311Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"9173dd6a-e883-4f2e-aaae-e8645eac7d3a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:35:11.311Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"9173dd6a-e883-4f2e-aaae-e8645eac7d3a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:35:11.312Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"9173dd6a-e883-4f2e-aaae-e8645eac7d3a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:35:11.312Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"9173dd6a-e883-4f2e-aaae-e8645eac7d3a","seconds":300}
{"level":"info","ts":"2026-06-05T09:35:11.320Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"68ebaf65-aec8-4ac4-b4ff-a30be116047c"}
{"level":"info","ts":"2026-06-05T09:35:11.320Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"68ebaf65-aec8-4ac4-b4ff-a30be116047c"}
{"level":"info","ts":"2026-06-05T09:35:11.320Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"68ebaf65-aec8-4ac4-b4ff-a30be116047c","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-05T09:35:11.320Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"68ebaf65-aec8-4ac4-b4ff-a30be116047c","Found 1 head Pod":"raycluster-auth-head-k2fq2","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:35:11.320Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"68ebaf65-aec8-4ac4-b4ff-a30be116047c","head Pod":"raycluster-auth-head-k2fq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-k2fq2. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:35:11.320Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"68ebaf65-aec8-4ac4-b4ff-a30be116047c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:35:11.320Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"68ebaf65-aec8-4ac4-b4ff-a30be116047c","worker Pod":"raycluster-auth-small-group-worker-w5wpq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-w5wpq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:35:11.320Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"68ebaf65-aec8-4ac4-b4ff-a30be116047c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:35:11.320Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"68ebaf65-aec8-4ac4-b4ff-a30be116047c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:35:11.320Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"68ebaf65-aec8-4ac4-b4ff-a30be116047c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:35:11.321Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"68ebaf65-aec8-4ac4-b4ff-a30be116047c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:35:11.321Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"68ebaf65-aec8-4ac4-b4ff-a30be116047c","seconds":300}
{"level":"info","ts":"2026-06-05T09:35:11.347Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"13e0cafe-c1cd-467b-bf20-16e20a8bc9e3"}
{"level":"info","ts":"2026-06-05T09:35:11.347Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"13e0cafe-c1cd-467b-bf20-16e20a8bc9e3"}
{"level":"info","ts":"2026-06-05T09:35:11.347Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"13e0cafe-c1cd-467b-bf20-16e20a8bc9e3","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-05T09:35:11.347Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"13e0cafe-c1cd-467b-bf20-16e20a8bc9e3","Found 1 head Pod":"raycluster-auth-head-k2fq2","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:35:11.347Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"13e0cafe-c1cd-467b-bf20-16e20a8bc9e3","head Pod":"raycluster-auth-head-k2fq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-k2fq2. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:35:11.347Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"13e0cafe-c1cd-467b-bf20-16e20a8bc9e3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:35:11.347Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"13e0cafe-c1cd-467b-bf20-16e20a8bc9e3","worker Pod":"raycluster-auth-small-group-worker-w5wpq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-w5wpq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:35:11.347Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"13e0cafe-c1cd-467b-bf20-16e20a8bc9e3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:35:11.347Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"13e0cafe-c1cd-467b-bf20-16e20a8bc9e3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:35:11.347Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"13e0cafe-c1cd-467b-bf20-16e20a8bc9e3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:35:11.347Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"13e0cafe-c1cd-467b-bf20-16e20a8bc9e3","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:35:11.347Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"13e0cafe-c1cd-467b-bf20-16e20a8bc9e3","seconds":300}
{"level":"info","ts":"2026-06-05T09:35:11.350Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"a5439ab7-505c-47ad-807f-21fc010283a3"}
{"level":"info","ts":"2026-06-05T09:35:11.350Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"a5439ab7-505c-47ad-807f-21fc010283a3"}
{"level":"info","ts":"2026-06-05T09:35:11.350Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"a5439ab7-505c-47ad-807f-21fc010283a3","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-05T09:35:11.350Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"a5439ab7-505c-47ad-807f-21fc010283a3","Found 1 head Pod":"raycluster-auth-head-k2fq2","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:35:11.350Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"a5439ab7-505c-47ad-807f-21fc010283a3","head Pod":"raycluster-auth-head-k2fq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-k2fq2. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:35:11.350Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"a5439ab7-505c-47ad-807f-21fc010283a3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:35:11.350Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"a5439ab7-505c-47ad-807f-21fc010283a3","worker Pod":"raycluster-auth-small-group-worker-w5wpq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-w5wpq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:35:11.350Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"a5439ab7-505c-47ad-807f-21fc010283a3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:35:11.350Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"a5439ab7-505c-47ad-807f-21fc010283a3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:35:11.350Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"a5439ab7-505c-47ad-807f-21fc010283a3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:35:11.351Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"a5439ab7-505c-47ad-807f-21fc010283a3","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:35:11.351Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"a5439ab7-505c-47ad-807f-21fc010283a3","seconds":300}
{"level":"info","ts":"2026-06-05T09:35:11.993Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"070ab7ef-a48c-478d-8c2a-e2a82e4d24a2"}
{"level":"info","ts":"2026-06-05T09:35:11.993Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"070ab7ef-a48c-478d-8c2a-e2a82e4d24a2"}
{"level":"info","ts":"2026-06-05T09:35:11.993Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"070ab7ef-a48c-478d-8c2a-e2a82e4d24a2","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-05T09:35:11.993Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"070ab7ef-a48c-478d-8c2a-e2a82e4d24a2","Found 1 head Pod":"raycluster-auth-head-k2fq2","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:35:11.993Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"070ab7ef-a48c-478d-8c2a-e2a82e4d24a2","head Pod":"raycluster-auth-head-k2fq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-k2fq2. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:35:11.993Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"070ab7ef-a48c-478d-8c2a-e2a82e4d24a2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:35:11.993Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"070ab7ef-a48c-478d-8c2a-e2a82e4d24a2","worker Pod":"raycluster-auth-small-group-worker-w5wpq","shouldDelete":true,"reason":"The worker Pod raycluster-auth-small-group-worker-w5wpq status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"error","ts":"2026-06-05T09:35:11.999Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"070ab7ef-a48c-478d-8c2a-e2a82e4d24a2","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"delete 1 unhealthy worker Pods","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:35:11.999Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"070ab7ef-a48c-478d-8c2a-e2a82e4d24a2","error":"delete 1 unhealthy worker Pods"}
{"level":"info","ts":"2026-06-05T09:35:11.999Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"070ab7ef-a48c-478d-8c2a-e2a82e4d24a2","oldReadyWorkerReplicas":1,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":1,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-05T09:35:11.999Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"070ab7ef-a48c-478d-8c2a-e2a82e4d24a2","name":"raycluster-auth","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:35:05Z","stateTransitionTimes":{"ready":"2026-06-05T09:35:05Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.14","serviceIP":"10.132.0.14","podName":"raycluster-auth-head-k2fq2","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:35:03Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:35:05Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:30:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:35:11Z","stateTransitionTimes":{"ready":"2026-06-05T09:35:05Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.14","serviceIP":"10.132.0.14","podName":"raycluster-auth-head-k2fq2","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:35:03Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:35:05Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:30:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:35:12.015Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"070ab7ef-a48c-478d-8c2a-e2a82e4d24a2"}
{"level":"error","ts":"2026-06-05T09:35:12.016Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"070ab7ef-a48c-478d-8c2a-e2a82e4d24a2","error":"delete 1 unhealthy worker Pods","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:35:12.016Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"48c945d2-c262-4601-802c-3ec67f750d0c","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:35:12.016Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"b0b438ec-f1be-41d1-b6ec-22d7d3637173","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:35:12.016Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"48c945d2-c262-4601-802c-3ec67f750d0c","pod":"raycluster-auth-head-k2fq2","ip":"10.132.0.14"}
{"level":"info","ts":"2026-06-05T09:35:12.016Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"48c945d2-c262-4601-802c-3ec67f750d0c","pod":"raycluster-auth-small-group-worker-w5wpq","ip":"10.133.0.34"}
{"level":"info","ts":"2026-06-05T09:35:12.016Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"48c945d2-c262-4601-802c-3ec67f750d0c","rayCluster":"raycluster-auth","podIPs":["10.132.0.14","10.133.0.34"]}
{"level":"info","ts":"2026-06-05T09:35:12.021Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"40f607cc-9bd9-4d0e-859a-71c17e303b09"}
{"level":"info","ts":"2026-06-05T09:35:12.021Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"40f607cc-9bd9-4d0e-859a-71c17e303b09"}
{"level":"info","ts":"2026-06-05T09:35:12.021Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"40f607cc-9bd9-4d0e-859a-71c17e303b09","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-05T09:35:12.022Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"40f607cc-9bd9-4d0e-859a-71c17e303b09","Found 1 head Pod":"raycluster-auth-head-k2fq2","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:35:12.022Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"40f607cc-9bd9-4d0e-859a-71c17e303b09","head Pod":"raycluster-auth-head-k2fq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-k2fq2. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:35:12.022Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"40f607cc-9bd9-4d0e-859a-71c17e303b09","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-06-05T09:35:12.022Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"40f607cc-9bd9-4d0e-859a-71c17e303b09","podName":"raycluster-auth-small-group-worker-w5wpq","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-06-05T09:35:12.022Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"40f607cc-9bd9-4d0e-859a-71c17e303b09","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:35:12.022Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"40f607cc-9bd9-4d0e-859a-71c17e303b09","seconds":300}
{"level":"info","ts":"2026-06-05T09:35:12.022Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"b0b438ec-f1be-41d1-b6ec-22d7d3637173","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:35:12.415Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"056d7bc7-8e0b-45bd-a3e2-62716a09290d","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:35:12.415Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"056d7bc7-8e0b-45bd-a3e2-62716a09290d","pod":"raycluster-auth-head-k2fq2","ip":"10.132.0.14"}
{"level":"info","ts":"2026-06-05T09:35:12.416Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"056d7bc7-8e0b-45bd-a3e2-62716a09290d","pod":"raycluster-auth-small-group-worker-w5wpq","ip":"10.133.0.34"}
{"level":"info","ts":"2026-06-05T09:35:12.416Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"056d7bc7-8e0b-45bd-a3e2-62716a09290d","rayCluster":"raycluster-auth","podIPs":["10.132.0.14","10.133.0.34"]}
{"level":"info","ts":"2026-06-05T09:35:12.939Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"6e5c3987-d0f6-4ab1-8c31-8cf175107892"}
{"level":"info","ts":"2026-06-05T09:35:12.939Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"6e5c3987-d0f6-4ab1-8c31-8cf175107892"}
{"level":"info","ts":"2026-06-05T09:35:12.939Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"6e5c3987-d0f6-4ab1-8c31-8cf175107892","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-05T09:35:12.940Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"6e5c3987-d0f6-4ab1-8c31-8cf175107892","Found 1 head Pod":"raycluster-auth-head-k2fq2","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:35:12.940Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"6e5c3987-d0f6-4ab1-8c31-8cf175107892","head Pod":"raycluster-auth-head-k2fq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-k2fq2. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:35:12.940Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"6e5c3987-d0f6-4ab1-8c31-8cf175107892","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-06-05T09:35:12.940Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"6e5c3987-d0f6-4ab1-8c31-8cf175107892","podName":"raycluster-auth-small-group-worker-w5wpq","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-06-05T09:35:12.940Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"6e5c3987-d0f6-4ab1-8c31-8cf175107892","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:35:12.940Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"6e5c3987-d0f6-4ab1-8c31-8cf175107892","seconds":300}
{"level":"info","ts":"2026-06-05T09:35:12.943Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"89a1c5fa-8210-4759-8a3c-72ed4cc3dfb7"}
{"level":"info","ts":"2026-06-05T09:35:12.943Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"89a1c5fa-8210-4759-8a3c-72ed4cc3dfb7"}
{"level":"info","ts":"2026-06-05T09:35:12.943Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"89a1c5fa-8210-4759-8a3c-72ed4cc3dfb7","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-05T09:35:12.943Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"89a1c5fa-8210-4759-8a3c-72ed4cc3dfb7","Found 1 head Pod":"raycluster-auth-head-k2fq2","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:35:12.943Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"89a1c5fa-8210-4759-8a3c-72ed4cc3dfb7","head Pod":"raycluster-auth-head-k2fq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-k2fq2. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:35:12.943Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"89a1c5fa-8210-4759-8a3c-72ed4cc3dfb7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:35:12.943Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"89a1c5fa-8210-4759-8a3c-72ed4cc3dfb7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:35:12.943Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"89a1c5fa-8210-4759-8a3c-72ed4cc3dfb7","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-05T09:35:12.943Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"89a1c5fa-8210-4759-8a3c-72ed4cc3dfb7","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-05T09:35:12.943Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"89a1c5fa-8210-4759-8a3c-72ed4cc3dfb7","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-05T09:35:12.944Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"89a1c5fa-8210-4759-8a3c-72ed4cc3dfb7"}
{"level":"info","ts":"2026-06-05T09:35:12.944Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"89a1c5fa-8210-4759-8a3c-72ed4cc3dfb7","nodeType":"worker","rayStartParams":{"address":"raycluster-auth-head-svc.test-ns-69tjt.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-05T09:35:12.944Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"89a1c5fa-8210-4759-8a3c-72ed4cc3dfb7","rayStartCmd":"ray start  --address=raycluster-auth-head-svc.test-ns-69tjt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:35:12.944Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"89a1c5fa-8210-4759-8a3c-72ed4cc3dfb7","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-auth-head-svc.test-ns-69tjt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:35:12.944Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"89a1c5fa-8210-4759-8a3c-72ed4cc3dfb7","enabled":true}
{"level":"error","ts":"2026-06-05T09:35:12.948Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"89a1c5fa-8210-4759-8a3c-72ed4cc3dfb7","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-69tjt because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:35:12.948Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"89a1c5fa-8210-4759-8a3c-72ed4cc3dfb7","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-69tjt because it is being terminated"}
{"level":"info","ts":"2026-06-05T09:35:12.948Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"89a1c5fa-8210-4759-8a3c-72ed4cc3dfb7","old conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:35:03Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:35:05Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:30:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:35:03Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:35:05Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:30:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-05T09:35:12Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-69tjt because it is being terminated"}]}
{"level":"info","ts":"2026-06-05T09:35:12.948Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"89a1c5fa-8210-4759-8a3c-72ed4cc3dfb7","name":"raycluster-auth","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:35:11Z","stateTransitionTimes":{"ready":"2026-06-05T09:35:05Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.14","serviceIP":"10.132.0.14","podName":"raycluster-auth-head-k2fq2","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:35:03Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:35:05Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:30:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:35:12Z","stateTransitionTimes":{"ready":"2026-06-05T09:35:05Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.14","serviceIP":"10.132.0.14","podName":"raycluster-auth-head-k2fq2","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:35:03Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:35:05Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:30:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-05T09:35:12Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-69tjt because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:35:13.009Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"3795ac94-5635-4ed2-b058-fc0b1c02c358","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:35:13.009Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"89a1c5fa-8210-4759-8a3c-72ed4cc3dfb7"}
{"level":"error","ts":"2026-06-05T09:35:13.009Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"89a1c5fa-8210-4759-8a3c-72ed4cc3dfb7","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-69tjt because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:35:13.009Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"c551a206-1028-44bf-9161-116c657c8b22","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:35:13.009Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"c551a206-1028-44bf-9161-116c657c8b22","pod":"raycluster-auth-head-k2fq2","ip":"10.132.0.14"}
{"level":"info","ts":"2026-06-05T09:35:13.014Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"95ddb1d8-c565-4a61-a6d7-0107cdac64cc"}
{"level":"info","ts":"2026-06-05T09:35:13.015Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"95ddb1d8-c565-4a61-a6d7-0107cdac64cc"}
{"level":"info","ts":"2026-06-05T09:35:13.015Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"95ddb1d8-c565-4a61-a6d7-0107cdac64cc","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-05T09:35:13.015Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"95ddb1d8-c565-4a61-a6d7-0107cdac64cc","Found 1 head Pod":"raycluster-auth-head-k2fq2","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:35:13.015Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"95ddb1d8-c565-4a61-a6d7-0107cdac64cc","head Pod":"raycluster-auth-head-k2fq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-k2fq2. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:35:13.015Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"95ddb1d8-c565-4a61-a6d7-0107cdac64cc","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:35:13.015Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"95ddb1d8-c565-4a61-a6d7-0107cdac64cc","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:35:13.015Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"95ddb1d8-c565-4a61-a6d7-0107cdac64cc","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-05T09:35:13.015Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"95ddb1d8-c565-4a61-a6d7-0107cdac64cc","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-05T09:35:13.015Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"95ddb1d8-c565-4a61-a6d7-0107cdac64cc","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-05T09:35:13.015Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"95ddb1d8-c565-4a61-a6d7-0107cdac64cc"}
{"level":"info","ts":"2026-06-05T09:35:13.015Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"95ddb1d8-c565-4a61-a6d7-0107cdac64cc","nodeType":"worker","rayStartParams":{"address":"raycluster-auth-head-svc.test-ns-69tjt.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-05T09:35:13.015Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"95ddb1d8-c565-4a61-a6d7-0107cdac64cc","rayStartCmd":"ray start  --address=raycluster-auth-head-svc.test-ns-69tjt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:35:13.015Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"95ddb1d8-c565-4a61-a6d7-0107cdac64cc","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-auth-head-svc.test-ns-69tjt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:35:13.015Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"95ddb1d8-c565-4a61-a6d7-0107cdac64cc","enabled":true}
{"level":"error","ts":"2026-06-05T09:35:13.025Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"95ddb1d8-c565-4a61-a6d7-0107cdac64cc","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-69tjt because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:35:13.025Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"95ddb1d8-c565-4a61-a6d7-0107cdac64cc","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-69tjt because it is being terminated"}
{"level":"info","ts":"2026-06-05T09:35:13.025Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"3795ac94-5635-4ed2-b058-fc0b1c02c358","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:35:13.025Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"95ddb1d8-c565-4a61-a6d7-0107cdac64cc"}
{"level":"error","ts":"2026-06-05T09:35:13.025Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"95ddb1d8-c565-4a61-a6d7-0107cdac64cc","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-69tjt because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:35:13.035Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"bd8e9852-129c-4e96-9c7a-18ccb73a02d4"}
{"level":"info","ts":"2026-06-05T09:35:13.035Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"bd8e9852-129c-4e96-9c7a-18ccb73a02d4"}
{"level":"info","ts":"2026-06-05T09:35:13.035Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"bd8e9852-129c-4e96-9c7a-18ccb73a02d4","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-05T09:35:13.035Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"bd8e9852-129c-4e96-9c7a-18ccb73a02d4","Found 1 head Pod":"raycluster-auth-head-k2fq2","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:35:13.035Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"bd8e9852-129c-4e96-9c7a-18ccb73a02d4","head Pod":"raycluster-auth-head-k2fq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-k2fq2. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:35:13.035Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"bd8e9852-129c-4e96-9c7a-18ccb73a02d4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:35:13.035Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"bd8e9852-129c-4e96-9c7a-18ccb73a02d4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:35:13.035Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"bd8e9852-129c-4e96-9c7a-18ccb73a02d4","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-05T09:35:13.035Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"bd8e9852-129c-4e96-9c7a-18ccb73a02d4","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-05T09:35:13.035Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"bd8e9852-129c-4e96-9c7a-18ccb73a02d4","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-05T09:35:13.035Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"bd8e9852-129c-4e96-9c7a-18ccb73a02d4"}
{"level":"info","ts":"2026-06-05T09:35:13.035Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"bd8e9852-129c-4e96-9c7a-18ccb73a02d4","nodeType":"worker","rayStartParams":{"address":"raycluster-auth-head-svc.test-ns-69tjt.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-05T09:35:13.035Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"bd8e9852-129c-4e96-9c7a-18ccb73a02d4","rayStartCmd":"ray start  --address=raycluster-auth-head-svc.test-ns-69tjt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:35:13.035Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"bd8e9852-129c-4e96-9c7a-18ccb73a02d4","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-auth-head-svc.test-ns-69tjt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:35:13.035Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"bd8e9852-129c-4e96-9c7a-18ccb73a02d4","enabled":true}
{"level":"error","ts":"2026-06-05T09:35:13.052Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"bd8e9852-129c-4e96-9c7a-18ccb73a02d4","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-69tjt because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:35:13.052Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"bd8e9852-129c-4e96-9c7a-18ccb73a02d4","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-69tjt because it is being terminated"}
{"level":"info","ts":"2026-06-05T09:35:13.052Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"c551a206-1028-44bf-9161-116c657c8b22","certificate":"ray-head-cert-raycluster-auth","dnsNames":["raycluster-auth-head-svc","localhost","raycluster-auth-head-svc.test-ns-69tjt.svc","raycluster-auth-head-svc.test-ns-69tjt.svc.cluster.local"],"ipAddresses":["10.132.0.14","127.0.0.1"]}
{"level":"info","ts":"2026-06-05T09:35:13.088Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"c551a206-1028-44bf-9161-116c657c8b22","certificate":"ray-worker-cert-raycluster-auth","dnsNames":["raycluster-auth-worker-svc","localhost","raycluster-auth-small-group","raycluster-auth-worker-svc.test-ns-69tjt.svc","raycluster-auth-worker-svc.test-ns-69tjt.svc.cluster.local","*.raycluster-auth-worker-svc.test-ns-69tjt.svc","*.raycluster-auth-worker-svc.test-ns-69tjt.svc.cluster.local","*-worker-*.test-ns-69tjt.svc","*-worker-*.test-ns-69tjt.svc.cluster.local"],"ipAddresses":["10.132.0.14","127.0.0.1"]}
{"level":"info","ts":"2026-06-05T09:35:13.088Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"c551a206-1028-44bf-9161-116c657c8b22"}
{"level":"info","ts":"2026-06-05T09:35:13.106Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"bd8e9852-129c-4e96-9c7a-18ccb73a02d4"}
{"level":"error","ts":"2026-06-05T09:35:13.106Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"bd8e9852-129c-4e96-9c7a-18ccb73a02d4","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-69tjt because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:35:13.126Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4fb0b046-31e2-4978-a89f-7c817d218168"}
{"level":"info","ts":"2026-06-05T09:35:13.126Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4fb0b046-31e2-4978-a89f-7c817d218168"}
{"level":"info","ts":"2026-06-05T09:35:13.126Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4fb0b046-31e2-4978-a89f-7c817d218168","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-05T09:35:13.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4fb0b046-31e2-4978-a89f-7c817d218168","Found 1 head Pod":"raycluster-auth-head-k2fq2","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:35:13.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4fb0b046-31e2-4978-a89f-7c817d218168","head Pod":"raycluster-auth-head-k2fq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-k2fq2. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:35:13.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4fb0b046-31e2-4978-a89f-7c817d218168","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:35:13.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4fb0b046-31e2-4978-a89f-7c817d218168","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:35:13.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4fb0b046-31e2-4978-a89f-7c817d218168","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-05T09:35:13.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4fb0b046-31e2-4978-a89f-7c817d218168","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-05T09:35:13.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4fb0b046-31e2-4978-a89f-7c817d218168","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-05T09:35:13.126Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4fb0b046-31e2-4978-a89f-7c817d218168"}
{"level":"info","ts":"2026-06-05T09:35:13.126Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4fb0b046-31e2-4978-a89f-7c817d218168","nodeType":"worker","rayStartParams":{"address":"raycluster-auth-head-svc.test-ns-69tjt.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-05T09:35:13.126Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4fb0b046-31e2-4978-a89f-7c817d218168","rayStartCmd":"ray start  --address=raycluster-auth-head-svc.test-ns-69tjt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:35:13.126Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4fb0b046-31e2-4978-a89f-7c817d218168","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-auth-head-svc.test-ns-69tjt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:35:13.126Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4fb0b046-31e2-4978-a89f-7c817d218168","enabled":true}
{"level":"error","ts":"2026-06-05T09:35:13.134Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4fb0b046-31e2-4978-a89f-7c817d218168","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-69tjt because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:35:13.134Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4fb0b046-31e2-4978-a89f-7c817d218168","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-69tjt because it is being terminated"}
{"level":"info","ts":"2026-06-05T09:35:13.134Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4fb0b046-31e2-4978-a89f-7c817d218168"}
{"level":"error","ts":"2026-06-05T09:35:13.134Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4fb0b046-31e2-4978-a89f-7c817d218168","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-69tjt because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:35:13.175Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"25bc2160-4887-4545-8816-d084a9525b94"}
{"level":"info","ts":"2026-06-05T09:35:13.175Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"25bc2160-4887-4545-8816-d084a9525b94"}
{"level":"info","ts":"2026-06-05T09:35:13.175Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"25bc2160-4887-4545-8816-d084a9525b94","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-05T09:35:13.176Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"25bc2160-4887-4545-8816-d084a9525b94","Found 1 head Pod":"raycluster-auth-head-k2fq2","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:35:13.176Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"25bc2160-4887-4545-8816-d084a9525b94","head Pod":"raycluster-auth-head-k2fq2","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-k2fq2. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:35:13.176Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"25bc2160-4887-4545-8816-d084a9525b94","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:35:13.176Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"25bc2160-4887-4545-8816-d084a9525b94","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:35:13.176Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"25bc2160-4887-4545-8816-d084a9525b94","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-05T09:35:13.176Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"25bc2160-4887-4545-8816-d084a9525b94","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-05T09:35:13.176Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"25bc2160-4887-4545-8816-d084a9525b94","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-05T09:35:13.176Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"25bc2160-4887-4545-8816-d084a9525b94"}
{"level":"info","ts":"2026-06-05T09:35:13.176Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"25bc2160-4887-4545-8816-d084a9525b94","nodeType":"worker","rayStartParams":{"address":"raycluster-auth-head-svc.test-ns-69tjt.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-05T09:35:13.176Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"25bc2160-4887-4545-8816-d084a9525b94","rayStartCmd":"ray start  --address=raycluster-auth-head-svc.test-ns-69tjt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:35:13.176Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"25bc2160-4887-4545-8816-d084a9525b94","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-auth-head-svc.test-ns-69tjt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:35:13.176Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"25bc2160-4887-4545-8816-d084a9525b94","enabled":true}
{"level":"error","ts":"2026-06-05T09:35:13.179Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"25bc2160-4887-4545-8816-d084a9525b94","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-69tjt because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:35:13.179Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"25bc2160-4887-4545-8816-d084a9525b94","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-69tjt because it is being terminated"}
{"level":"info","ts":"2026-06-05T09:35:13.180Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"25bc2160-4887-4545-8816-d084a9525b94","old conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:35:03Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:35:05Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:30:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-05T09:35:12Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-69tjt because it is being terminated"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:35:13Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:35:05Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:30:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-05T09:35:12Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-69tjt because it is being terminated"}]}
{"level":"info","ts":"2026-06-05T09:35:13.180Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"25bc2160-4887-4545-8816-d084a9525b94","name":"raycluster-auth","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:35:12Z","stateTransitionTimes":{"ready":"2026-06-05T09:35:05Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.14","serviceIP":"10.132.0.14","podName":"raycluster-auth-head-k2fq2","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:35:03Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:35:05Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:30:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-05T09:35:12Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-69tjt because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:35:13Z","stateTransitionTimes":{"ready":"2026-06-05T09:35:05Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.14","serviceIP":"10.132.0.14","podName":"raycluster-auth-head-k2fq2","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:35:13Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:35:05Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:30:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-05T09:35:12Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-69tjt because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:35:13.192Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"36e59bb4-683e-4505-87ed-634af5942264","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:35:13.192Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"36e59bb4-683e-4505-87ed-634af5942264","pod":"raycluster-auth-head-k2fq2","ip":"10.132.0.14"}
{"level":"info","ts":"2026-06-05T09:35:13.192Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"36e59bb4-683e-4505-87ed-634af5942264"}
{"level":"info","ts":"2026-06-05T09:35:13.192Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"25bc2160-4887-4545-8816-d084a9525b94"}
{"level":"error","ts":"2026-06-05T09:35:13.192Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"25bc2160-4887-4545-8816-d084a9525b94","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-69tjt because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:35:13.192Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"5b95ccf1-01b3-44b9-8558-8956a3a2d0a8"}
{"level":"info","ts":"2026-06-05T09:35:13.192Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"5b95ccf1-01b3-44b9-8558-8956a3a2d0a8"}
{"level":"info","ts":"2026-06-05T09:35:13.192Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"5b95ccf1-01b3-44b9-8558-8956a3a2d0a8","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-05T09:35:13.192Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"5b95ccf1-01b3-44b9-8558-8956a3a2d0a8","Found 1 head Pod":"raycluster-auth-head-k2fq2","Pod status":"Failed","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"&ContainerStateTerminated{ExitCode:1,Signal:0,Reason:Error,Message:,StartedAt:2026-06-05 09:34:49 +0000 UTC,FinishedAt:2026-06-05 09:35:12 +0000 UTC,ContainerID:cri-o://b3de9974f488d0d16ff473b6dfc91c7358c7c2bc566816a862de03a4d10c5e06,}"}
{"level":"info","ts":"2026-06-05T09:35:13.192Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"5b95ccf1-01b3-44b9-8558-8956a3a2d0a8","head Pod":"raycluster-auth-head-k2fq2","shouldDelete":true,"reason":"The head Pod raycluster-auth-head-k2fq2 status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"info","ts":"2026-06-05T09:35:13.192Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"1f374366-3c2e-4961-a83b-52a17c9bb1bc","cluster":"raycluster-auth"}
{"level":"error","ts":"2026-06-05T09:35:13.206Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"5b95ccf1-01b3-44b9-8558-8956a3a2d0a8","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"The head Pod raycluster-auth-head-k2fq2 status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:35:13.206Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"5b95ccf1-01b3-44b9-8558-8956a3a2d0a8","error":"The head Pod raycluster-auth-head-k2fq2 status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"info","ts":"2026-06-05T09:35:13.206Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"5b95ccf1-01b3-44b9-8558-8956a3a2d0a8"}
{"level":"error","ts":"2026-06-05T09:35:13.206Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"5b95ccf1-01b3-44b9-8558-8956a3a2d0a8","error":"The head Pod raycluster-auth-head-k2fq2 status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:35:13.209Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"1f374366-3c2e-4961-a83b-52a17c9bb1bc","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:35:13.273Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"427819f0-526a-4872-ab1f-50a2a870b2a2"}
{"level":"info","ts":"2026-06-05T09:35:13.273Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"427819f0-526a-4872-ab1f-50a2a870b2a2"}
{"level":"info","ts":"2026-06-05T09:35:13.273Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"427819f0-526a-4872-ab1f-50a2a870b2a2","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-05T09:35:13.273Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"427819f0-526a-4872-ab1f-50a2a870b2a2","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-06-05T09:35:13.273Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"427819f0-526a-4872-ab1f-50a2a870b2a2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:35:13.273Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"427819f0-526a-4872-ab1f-50a2a870b2a2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:35:13.273Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"427819f0-526a-4872-ab1f-50a2a870b2a2","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-05T09:35:13.273Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"427819f0-526a-4872-ab1f-50a2a870b2a2","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-05T09:35:13.273Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"427819f0-526a-4872-ab1f-50a2a870b2a2","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-05T09:35:13.273Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"427819f0-526a-4872-ab1f-50a2a870b2a2"}
{"level":"info","ts":"2026-06-05T09:35:13.273Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"427819f0-526a-4872-ab1f-50a2a870b2a2","nodeType":"worker","rayStartParams":{"address":"raycluster-auth-head-svc.test-ns-69tjt.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-05T09:35:13.273Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"427819f0-526a-4872-ab1f-50a2a870b2a2","rayStartCmd":"ray start  --address=raycluster-auth-head-svc.test-ns-69tjt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:35:13.273Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"427819f0-526a-4872-ab1f-50a2a870b2a2","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-auth-head-svc.test-ns-69tjt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:35:13.273Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"427819f0-526a-4872-ab1f-50a2a870b2a2","enabled":true}
{"level":"error","ts":"2026-06-05T09:35:13.277Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"427819f0-526a-4872-ab1f-50a2a870b2a2","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-69tjt because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:35:13.277Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"427819f0-526a-4872-ab1f-50a2a870b2a2","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-69tjt because it is being terminated"}
{"level":"info","ts":"2026-06-05T09:35:13.277Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"427819f0-526a-4872-ab1f-50a2a870b2a2"}
{"level":"error","ts":"2026-06-05T09:35:13.277Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"427819f0-526a-4872-ab1f-50a2a870b2a2","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-69tjt because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:35:13.598Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"3560b26a-ed1e-42c6-bf54-23eab556814b"}
{"level":"info","ts":"2026-06-05T09:35:13.598Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"3560b26a-ed1e-42c6-bf54-23eab556814b"}
{"level":"info","ts":"2026-06-05T09:35:13.598Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"3560b26a-ed1e-42c6-bf54-23eab556814b","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-05T09:35:13.598Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"3560b26a-ed1e-42c6-bf54-23eab556814b","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-06-05T09:35:13.598Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"3560b26a-ed1e-42c6-bf54-23eab556814b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:35:13.598Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"3560b26a-ed1e-42c6-bf54-23eab556814b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:35:13.598Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"3560b26a-ed1e-42c6-bf54-23eab556814b","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-05T09:35:13.598Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"3560b26a-ed1e-42c6-bf54-23eab556814b","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-05T09:35:13.598Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"3560b26a-ed1e-42c6-bf54-23eab556814b","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-05T09:35:13.598Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"3560b26a-ed1e-42c6-bf54-23eab556814b"}
{"level":"info","ts":"2026-06-05T09:35:13.598Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"3560b26a-ed1e-42c6-bf54-23eab556814b","nodeType":"worker","rayStartParams":{"address":"raycluster-auth-head-svc.test-ns-69tjt.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-05T09:35:13.599Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"3560b26a-ed1e-42c6-bf54-23eab556814b","rayStartCmd":"ray start  --address=raycluster-auth-head-svc.test-ns-69tjt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:35:13.599Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"3560b26a-ed1e-42c6-bf54-23eab556814b","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-auth-head-svc.test-ns-69tjt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:35:13.599Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"3560b26a-ed1e-42c6-bf54-23eab556814b","enabled":true}
{"level":"error","ts":"2026-06-05T09:35:13.602Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"3560b26a-ed1e-42c6-bf54-23eab556814b","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-69tjt because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:35:13.603Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"3560b26a-ed1e-42c6-bf54-23eab556814b","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-69tjt because it is being terminated"}
{"level":"info","ts":"2026-06-05T09:35:13.603Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"3560b26a-ed1e-42c6-bf54-23eab556814b"}
{"level":"error","ts":"2026-06-05T09:35:13.603Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"3560b26a-ed1e-42c6-bf54-23eab556814b","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-69tjt because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:35:13.959Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"3353b16b-3e4f-447e-8b4b-8bc654228f7e"}
{"level":"info","ts":"2026-06-05T09:35:13.959Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"3353b16b-3e4f-447e-8b4b-8bc654228f7e"}
{"level":"info","ts":"2026-06-05T09:35:13.959Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"3353b16b-3e4f-447e-8b4b-8bc654228f7e","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-05T09:35:13.959Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"3353b16b-3e4f-447e-8b4b-8bc654228f7e","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-06-05T09:35:13.959Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"3353b16b-3e4f-447e-8b4b-8bc654228f7e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:35:13.959Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"3353b16b-3e4f-447e-8b4b-8bc654228f7e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:35:13.959Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"3353b16b-3e4f-447e-8b4b-8bc654228f7e","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-05T09:35:13.959Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"3353b16b-3e4f-447e-8b4b-8bc654228f7e","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-05T09:35:13.959Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"3353b16b-3e4f-447e-8b4b-8bc654228f7e","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-05T09:35:13.959Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"3353b16b-3e4f-447e-8b4b-8bc654228f7e"}
{"level":"info","ts":"2026-06-05T09:35:13.959Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"3353b16b-3e4f-447e-8b4b-8bc654228f7e","nodeType":"worker","rayStartParams":{"address":"raycluster-auth-head-svc.test-ns-69tjt.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-05T09:35:13.959Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"3353b16b-3e4f-447e-8b4b-8bc654228f7e","rayStartCmd":"ray start  --address=raycluster-auth-head-svc.test-ns-69tjt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:35:13.959Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"3353b16b-3e4f-447e-8b4b-8bc654228f7e","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-auth-head-svc.test-ns-69tjt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:35:13.959Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"3353b16b-3e4f-447e-8b4b-8bc654228f7e","enabled":true}
{"level":"error","ts":"2026-06-05T09:35:13.964Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"3353b16b-3e4f-447e-8b4b-8bc654228f7e","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-69tjt because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:35:13.964Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"3353b16b-3e4f-447e-8b4b-8bc654228f7e","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-69tjt because it is being terminated"}
{"level":"info","ts":"2026-06-05T09:35:13.964Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"3353b16b-3e4f-447e-8b4b-8bc654228f7e","filter labels":["test-ns-69tjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:35:13.965Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"3353b16b-3e4f-447e-8b4b-8bc654228f7e","filter labels":["test-ns-69tjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:35:13.965Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"3353b16b-3e4f-447e-8b4b-8bc654228f7e","filter labels":["test-ns-69tjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:35:13.965Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"3353b16b-3e4f-447e-8b4b-8bc654228f7e","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podIP":"10.132.0.14","serviceIP":"10.132.0.14","podName":"raycluster-auth-head-k2fq2","serviceName":"raycluster-auth-head-svc"},"newHead":{"serviceName":"raycluster-auth-head-svc"}}
{"level":"info","ts":"2026-06-05T09:35:13.965Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"3353b16b-3e4f-447e-8b4b-8bc654228f7e","name":"raycluster-auth","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:35:13Z","stateTransitionTimes":{"ready":"2026-06-05T09:35:05Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.14","serviceIP":"10.132.0.14","podName":"raycluster-auth-head-k2fq2","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:35:13Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:35:05Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:30:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-05T09:35:12Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-69tjt because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:35:13Z","stateTransitionTimes":{"ready":"2026-06-05T09:35:05Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:35:13Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:35:05Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:30:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-05T09:35:12Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-69tjt because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:35:13.979Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"3353b16b-3e4f-447e-8b4b-8bc654228f7e"}
{"level":"error","ts":"2026-06-05T09:35:13.979Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"3353b16b-3e4f-447e-8b4b-8bc654228f7e","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-69tjt because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:35:13.979Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4f0cff48-bfda-442d-a04d-cff377e328a4"}
{"level":"info","ts":"2026-06-05T09:35:13.979Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4f0cff48-bfda-442d-a04d-cff377e328a4"}
{"level":"info","ts":"2026-06-05T09:35:13.979Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"82723da8-a277-4315-896b-1322942b9599","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:35:13.979Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4f0cff48-bfda-442d-a04d-cff377e328a4","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-05T09:35:13.979Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"3eed33a7-6ed0-493e-ab32-850647762a66","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:35:13.979Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4f0cff48-bfda-442d-a04d-cff377e328a4"}
{"level":"info","ts":"2026-06-05T09:35:13.979Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4f0cff48-bfda-442d-a04d-cff377e328a4","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:35:13.979Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4f0cff48-bfda-442d-a04d-cff377e328a4"}
{"level":"info","ts":"2026-06-05T09:35:13.979Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4f0cff48-bfda-442d-a04d-cff377e328a4","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:35:13.979Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4f0cff48-bfda-442d-a04d-cff377e328a4","cluster":"raycluster-auth","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:35:13.979Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4f0cff48-bfda-442d-a04d-cff377e328a4","cluster":"raycluster-auth","authType":"IntegratedOAuth","serviceAccount":"raycluster-auth-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-05T09:35:13.979Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4f0cff48-bfda-442d-a04d-cff377e328a4","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"raycluster-auth","ray.io/group":"headgroup","ray.io/identifier":"raycluster-auth-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-05T09:35:13.979Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4f0cff48-bfda-442d-a04d-cff377e328a4","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"raycluster-auth-head-svc.test-ns-69tjt.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-05T09:35:13.979Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4f0cff48-bfda-442d-a04d-cff377e328a4","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-69tjt.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-05T09:35:13.979Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4f0cff48-bfda-442d-a04d-cff377e328a4","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-69tjt.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-05T09:35:13.979Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4f0cff48-bfda-442d-a04d-cff377e328a4","enabled":true}
{"level":"error","ts":"2026-06-05T09:35:13.984Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4f0cff48-bfda-442d-a04d-cff377e328a4","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-69tjt because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:35:13.984Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4f0cff48-bfda-442d-a04d-cff377e328a4","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-69tjt because it is being terminated"}
{"level":"info","ts":"2026-06-05T09:35:13.984Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4f0cff48-bfda-442d-a04d-cff377e328a4","filter labels":["test-ns-69tjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:35:13.984Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4f0cff48-bfda-442d-a04d-cff377e328a4","filter labels":["test-ns-69tjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:35:13.984Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4f0cff48-bfda-442d-a04d-cff377e328a4","filter labels":["test-ns-69tjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:35:13.984Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4f0cff48-bfda-442d-a04d-cff377e328a4","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:35:13Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:35:05Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:30:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-05T09:35:12Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-69tjt because it is being terminated"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:35:13Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:35:05Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:30:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-05T09:35:12Z","reason":"FailedCreateHeadPod","message":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-69tjt because it is being terminated"}]}
{"level":"info","ts":"2026-06-05T09:35:13.984Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4f0cff48-bfda-442d-a04d-cff377e328a4","name":"raycluster-auth","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:35:13Z","stateTransitionTimes":{"ready":"2026-06-05T09:35:05Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:35:13Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:35:05Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:30:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-05T09:35:12Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-69tjt because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:35:13Z","stateTransitionTimes":{"ready":"2026-06-05T09:35:05Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:35:13Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:35:05Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:30:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:30:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-05T09:35:12Z","reason":"FailedCreateHeadPod","message":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-69tjt because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:35:13.985Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"3eed33a7-6ed0-493e-ab32-850647762a66","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:35:13.994Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"82723da8-a277-4315-896b-1322942b9599","certificate":"ray-head-cert-raycluster-auth","dnsNames":["raycluster-auth-head-svc","localhost","raycluster-auth-head-svc.test-ns-69tjt.svc","raycluster-auth-head-svc.test-ns-69tjt.svc.cluster.local"],"ipAddresses":["127.0.0.1"]}
{"level":"info","ts":"2026-06-05T09:35:14.001Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"338e66ec-4cf5-49c2-87d0-28e255ff5e37","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:35:14.001Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4f0cff48-bfda-442d-a04d-cff377e328a4"}
{"level":"error","ts":"2026-06-05T09:35:14.001Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4f0cff48-bfda-442d-a04d-cff377e328a4","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-69tjt because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:35:14.008Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"338e66ec-4cf5-49c2-87d0-28e255ff5e37","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:35:14.011Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"82723da8-a277-4315-896b-1322942b9599","certificate":"ray-worker-cert-raycluster-auth","dnsNames":["raycluster-auth-worker-svc","localhost","raycluster-auth-small-group","raycluster-auth-worker-svc.test-ns-69tjt.svc","raycluster-auth-worker-svc.test-ns-69tjt.svc.cluster.local","*.raycluster-auth-worker-svc.test-ns-69tjt.svc","*.raycluster-auth-worker-svc.test-ns-69tjt.svc.cluster.local","*-worker-*.test-ns-69tjt.svc","*-worker-*.test-ns-69tjt.svc.cluster.local"],"ipAddresses":["127.0.0.1"]}
{"level":"info","ts":"2026-06-05T09:35:14.011Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"82723da8-a277-4315-896b-1322942b9599"}
{"level":"info","ts":"2026-06-05T09:35:14.011Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"25efb45c-ba63-4c9b-930f-2d0955172dfd","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:35:14.011Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"25efb45c-ba63-4c9b-930f-2d0955172dfd"}
{"level":"info","ts":"2026-06-05T09:35:14.244Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4db107de-a293-487a-95c3-30ba9946d511"}
{"level":"info","ts":"2026-06-05T09:35:14.244Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4db107de-a293-487a-95c3-30ba9946d511"}
{"level":"info","ts":"2026-06-05T09:35:14.244Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4db107de-a293-487a-95c3-30ba9946d511","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-05T09:35:14.244Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4db107de-a293-487a-95c3-30ba9946d511"}
{"level":"info","ts":"2026-06-05T09:35:14.244Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4db107de-a293-487a-95c3-30ba9946d511","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:35:14.244Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4db107de-a293-487a-95c3-30ba9946d511"}
{"level":"info","ts":"2026-06-05T09:35:14.244Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4db107de-a293-487a-95c3-30ba9946d511","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:35:14.244Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4db107de-a293-487a-95c3-30ba9946d511","cluster":"raycluster-auth","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:35:14.244Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4db107de-a293-487a-95c3-30ba9946d511","cluster":"raycluster-auth","authType":"IntegratedOAuth","serviceAccount":"raycluster-auth-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-05T09:35:14.244Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4db107de-a293-487a-95c3-30ba9946d511","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"raycluster-auth","ray.io/group":"headgroup","ray.io/identifier":"raycluster-auth-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-05T09:35:14.244Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4db107de-a293-487a-95c3-30ba9946d511","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"raycluster-auth-head-svc.test-ns-69tjt.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-05T09:35:14.244Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4db107de-a293-487a-95c3-30ba9946d511","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-69tjt.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-05T09:35:14.244Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4db107de-a293-487a-95c3-30ba9946d511","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-69tjt.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-05T09:35:14.244Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4db107de-a293-487a-95c3-30ba9946d511","enabled":true}
{"level":"error","ts":"2026-06-05T09:35:14.249Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4db107de-a293-487a-95c3-30ba9946d511","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-69tjt because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:35:14.249Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4db107de-a293-487a-95c3-30ba9946d511","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-69tjt because it is being terminated"}
{"level":"info","ts":"2026-06-05T09:35:14.249Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4db107de-a293-487a-95c3-30ba9946d511","filter labels":["test-ns-69tjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:35:14.249Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4db107de-a293-487a-95c3-30ba9946d511","filter labels":["test-ns-69tjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:35:14.249Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4db107de-a293-487a-95c3-30ba9946d511","filter labels":["test-ns-69tjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:35:14.249Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4db107de-a293-487a-95c3-30ba9946d511"}
{"level":"error","ts":"2026-06-05T09:35:14.249Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"4db107de-a293-487a-95c3-30ba9946d511","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-69tjt because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:35:16.388Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"3e10faa0-12a2-4a87-a35a-846082a3d1ce"}
{"level":"info","ts":"2026-06-05T09:35:16.388Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"1facd59a-869e-456e-aee2-81149b089abd","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-69tjt"}}
{"level":"info","ts":"2026-06-05T09:35:16.388Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"reconcileID":"86fd2bb5-45f1-4f35-bb3b-b5de91e3b31e"}
{"level":"info","ts":"2026-06-05T09:35:16.388Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"a040cfb1-673c-4b4a-910b-1ef6a8896c35","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:35:16.388Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"1facd59a-869e-456e-aee2-81149b089abd","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:35:16.388Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"1facd59a-869e-456e-aee2-81149b089abd","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:35:16.388Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"1facd59a-869e-456e-aee2-81149b089abd","namespace":"test-ns-69tjt","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-05T09:35:16.388Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"a040cfb1-673c-4b4a-910b-1ef6a8896c35"}
{"level":"info","ts":"2026-06-05T09:35:16.394Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"1facd59a-869e-456e-aee2-81149b089abd","name":"kuberay-gateway-access","namespace":"test-ns-69tjt"}
{"level":"info","ts":"2026-06-05T09:35:16.400Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"1facd59a-869e-456e-aee2-81149b089abd","name":"test-ns-69tjt-raycluster-auth","namespace":"opendatahub"}
{"level":"info","ts":"2026-06-05T09:35:16.405Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"1facd59a-869e-456e-aee2-81149b089abd","configMap":"kube-rbac-proxy-config-raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:35:16.410Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"1facd59a-869e-456e-aee2-81149b089abd","serviceAccount":"raycluster-auth-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:35:16.423Z","logger":"raycluster-resource","msg":"default","name":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:35:16.423Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"raycluster-auth","namespace":"test-ns-69tjt"}
{"level":"info","ts":"2026-06-05T09:35:16.437Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"1facd59a-869e-456e-aee2-81149b089abd","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:35:16.438Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"57021972-752c-46b7-b2c6-0a62e8cd0746","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-69tjt"}}
{"level":"info","ts":"2026-06-05T09:35:16.438Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"57021972-752c-46b7-b2c6-0a62e8cd0746"}
{"level":"info","ts":"2026-06-05T09:35:16.438Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"57021972-752c-46b7-b2c6-0a62e8cd0746","namespace":"test-ns-69tjt","authClusters":0}
{"level":"info","ts":"2026-06-05T09:35:16.438Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"57021972-752c-46b7-b2c6-0a62e8cd0746","namespace":"test-ns-69tjt"}
{"level":"info","ts":"2026-06-05T09:35:16.438Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"570f6773-1a2a-498a-8216-1b6188562ab0","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-69tjt"}}
{"level":"info","ts":"2026-06-05T09:35:16.438Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"570f6773-1a2a-498a-8216-1b6188562ab0"}
{"level":"info","ts":"2026-06-05T09:35:16.438Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"0f1bfb47-78d7-472b-a0a0-0b855cf365b5","namespace":"test-ns-69tjt","clusterName":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:35:16.438Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"570f6773-1a2a-498a-8216-1b6188562ab0","namespace":"test-ns-69tjt","authClusters":0}
{"level":"info","ts":"2026-06-05T09:35:16.438Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"570f6773-1a2a-498a-8216-1b6188562ab0","namespace":"test-ns-69tjt"}
{"level":"info","ts":"2026-06-05T09:35:16.438Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"dafa34d9-156b-4bb9-bd47-20217df01281"}
{"level":"info","ts":"2026-06-05T09:35:16.447Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"b71c3bbe-dfb1-4719-a468-734bc87f2193"}
{"level":"info","ts":"2026-06-05T09:35:16.454Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"15c4f823-aa5f-4fb7-86b8-e63a8348ce0d","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-69tjt"}}
{"level":"info","ts":"2026-06-05T09:35:16.454Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"15c4f823-aa5f-4fb7-86b8-e63a8348ce0d"}
{"level":"info","ts":"2026-06-05T09:35:16.454Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"15c4f823-aa5f-4fb7-86b8-e63a8348ce0d","namespace":"test-ns-69tjt","authClusters":0}
{"level":"info","ts":"2026-06-05T09:35:16.454Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"15c4f823-aa5f-4fb7-86b8-e63a8348ce0d","namespace":"test-ns-69tjt"}
{"level":"error","ts":"2026-06-05T09:35:16.484Z","msg":"Failed to delete worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"0f1bfb47-78d7-472b-a0a0-0b855cf365b5","certificate":"ray-worker-cert-raycluster-auth","error":"certificates.cert-manager.io \"ray-worker-cert-raycluster-auth\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:268\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:35:16.496Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"25847ab1-ed07-45b1-9aef-df3c34dcf66a"}
{"level":"info","ts":"2026-06-05T09:35:16.525Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"0f1bfb47-78d7-472b-a0a0-0b855cf365b5","errorCount":1}
{"level":"info","ts":"2026-06-05T09:35:16.525Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"0f1bfb47-78d7-472b-a0a0-0b855cf365b5"}
{"level":"error","ts":"2026-06-05T09:35:16.525Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"0f1bfb47-78d7-472b-a0a0-0b855cf365b5","error":"certificates.cert-manager.io \"ray-worker-cert-raycluster-auth\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:35:16.531Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"f14d887b-d702-4657-9cfc-b61961e88f4c","namespace":"test-ns-69tjt","clusterName":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:35:16.531Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-69tjt"},"namespace":"test-ns-69tjt","name":"raycluster-auth","reconcileID":"f14d887b-d702-4657-9cfc-b61961e88f4c","namespace":"test-ns-69tjt","clusterName":"raycluster-auth"}
{"level":"info","ts":"2026-06-05T09:35:36.517Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"c9c5f03a-c2a9-4e36-92ab-9562a061426a","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:35:36.518Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"c9c5f03a-c2a9-4e36-92ab-9562a061426a","pod":"raycluster-small-group-worker-j9pbw","ip":"10.133.0.35"}
{"level":"info","ts":"2026-06-05T09:35:36.532Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"c9c5f03a-c2a9-4e36-92ab-9562a061426a","certificate":"ray-head-cert-raycluster","dnsNames":["raycluster-head-svc","localhost","raycluster-head-svc.test-ns-rfwvh.svc","raycluster-head-svc.test-ns-rfwvh.svc.cluster.local"],"ipAddresses":["10.133.0.35","127.0.0.1"]}
{"level":"info","ts":"2026-06-05T09:35:36.545Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"c9c5f03a-c2a9-4e36-92ab-9562a061426a","certificate":"ray-worker-cert-raycluster","dnsNames":["raycluster-worker-svc","localhost","raycluster-small-group","raycluster-worker-svc.test-ns-rfwvh.svc","raycluster-worker-svc.test-ns-rfwvh.svc.cluster.local","*.raycluster-worker-svc.test-ns-rfwvh.svc","*.raycluster-worker-svc.test-ns-rfwvh.svc.cluster.local","*-worker-*.test-ns-rfwvh.svc","*-worker-*.test-ns-rfwvh.svc.cluster.local"],"ipAddresses":["10.133.0.35","127.0.0.1"]}
{"level":"info","ts":"2026-06-05T09:35:36.545Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"c9c5f03a-c2a9-4e36-92ab-9562a061426a","rayCluster":"raycluster","podIPs":["10.133.0.35"]}
{"level":"info","ts":"2026-06-05T09:36:36.545Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"fa534265-d161-4156-8a6e-cfd92191f057","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:36:36.546Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"fa534265-d161-4156-8a6e-cfd92191f057","pod":"raycluster-small-group-worker-j9pbw","ip":"10.133.0.35"}
{"level":"info","ts":"2026-06-05T09:36:36.546Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"fa534265-d161-4156-8a6e-cfd92191f057","rayCluster":"raycluster","podIPs":["10.133.0.35"]}
{"level":"info","ts":"2026-06-05T09:37:36.546Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"9e136c88-3376-4607-9701-b824be5098d6","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:37:36.546Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"9e136c88-3376-4607-9701-b824be5098d6","pod":"raycluster-small-group-worker-j9pbw","ip":"10.133.0.35"}
{"level":"info","ts":"2026-06-05T09:37:36.547Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"9e136c88-3376-4607-9701-b824be5098d6","rayCluster":"raycluster","podIPs":["10.133.0.35"]}
{"level":"info","ts":"2026-06-05T09:38:36.547Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"c8a688a4-c971-4b84-8731-c1e65b4ca28e","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:38:36.547Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"c8a688a4-c971-4b84-8731-c1e65b4ca28e","pod":"raycluster-small-group-worker-j9pbw","ip":"10.133.0.35"}
{"level":"info","ts":"2026-06-05T09:38:36.547Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"c8a688a4-c971-4b84-8731-c1e65b4ca28e","rayCluster":"raycluster","podIPs":["10.133.0.35"]}
{"level":"info","ts":"2026-06-05T09:38:39.712Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"fae42ef6-d8d1-445b-b32c-ea7058a22ac3"}
{"level":"info","ts":"2026-06-05T09:38:39.712Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"fae42ef6-d8d1-445b-b32c-ea7058a22ac3"}
{"level":"info","ts":"2026-06-05T09:38:39.712Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"fae42ef6-d8d1-445b-b32c-ea7058a22ac3","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-05T09:38:39.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"fae42ef6-d8d1-445b-b32c-ea7058a22ac3","Found 1 head Pod":"raycluster-head-vjd7d","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:38:39.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"fae42ef6-d8d1-445b-b32c-ea7058a22ac3","head Pod":"raycluster-head-vjd7d","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-vjd7d. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:38:39.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"fae42ef6-d8d1-445b-b32c-ea7058a22ac3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:38:39.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"fae42ef6-d8d1-445b-b32c-ea7058a22ac3","worker Pod":"raycluster-small-group-worker-j9pbw","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-j9pbw. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:38:39.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"fae42ef6-d8d1-445b-b32c-ea7058a22ac3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:38:39.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"fae42ef6-d8d1-445b-b32c-ea7058a22ac3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:38:39.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"fae42ef6-d8d1-445b-b32c-ea7058a22ac3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:38:39.712Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"fae42ef6-d8d1-445b-b32c-ea7058a22ac3","podName":"raycluster-head-vjd7d","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:38:39.712Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"fae42ef6-d8d1-445b-b32c-ea7058a22ac3","podName":"raycluster-head-vjd7d","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:38:39.712Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"fae42ef6-d8d1-445b-b32c-ea7058a22ac3","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"raycluster-head-vjd7d","serviceName":"raycluster-head-svc"},"newHead":{"podIP":"10.134.0.20","serviceIP":"10.134.0.20","podName":"raycluster-head-vjd7d","serviceName":"raycluster-head-svc"}}
{"level":"info","ts":"2026-06-05T09:38:39.712Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"fae42ef6-d8d1-445b-b32c-ea7058a22ac3","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:35:09Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-head-vjd7d","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:38:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.20","serviceIP":"10.134.0.20","podName":"raycluster-head-vjd7d","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:38:39.723Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"6f75934c-9822-4b02-99df-d400bba577f2","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:38:39.723Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"5d4f0a62-5d3c-4b89-af7a-e8babd3f5caf","cluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:38:39.723Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"6f75934c-9822-4b02-99df-d400bba577f2","pod":"raycluster-head-vjd7d","ip":"10.134.0.20"}
{"level":"info","ts":"2026-06-05T09:38:39.723Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"6f75934c-9822-4b02-99df-d400bba577f2","pod":"raycluster-small-group-worker-j9pbw","ip":"10.133.0.35"}
{"level":"info","ts":"2026-06-05T09:38:39.741Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"5d4f0a62-5d3c-4b89-af7a-e8babd3f5caf","name":"raycluster-head"}
{"level":"info","ts":"2026-06-05T09:38:39.750Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"6f75934c-9822-4b02-99df-d400bba577f2","certificate":"ray-head-cert-raycluster","dnsNames":["raycluster-head-svc","localhost","raycluster-head-svc.test-ns-rfwvh.svc","raycluster-head-svc.test-ns-rfwvh.svc.cluster.local"],"ipAddresses":["10.134.0.20","10.133.0.35","127.0.0.1"]}
{"level":"info","ts":"2026-06-05T09:38:39.754Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"5d4f0a62-5d3c-4b89-af7a-e8babd3f5caf","name":"raycluster-workers"}
{"level":"info","ts":"2026-06-05T09:38:39.754Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"5d4f0a62-5d3c-4b89-af7a-e8babd3f5caf","cluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:38:39.764Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"6f75934c-9822-4b02-99df-d400bba577f2","certificate":"ray-worker-cert-raycluster","dnsNames":["raycluster-worker-svc","localhost","raycluster-small-group","raycluster-worker-svc.test-ns-rfwvh.svc","raycluster-worker-svc.test-ns-rfwvh.svc.cluster.local","*.raycluster-worker-svc.test-ns-rfwvh.svc","*.raycluster-worker-svc.test-ns-rfwvh.svc.cluster.local","*-worker-*.test-ns-rfwvh.svc","*-worker-*.test-ns-rfwvh.svc.cluster.local"],"ipAddresses":["10.134.0.20","10.133.0.35","127.0.0.1"]}
{"level":"info","ts":"2026-06-05T09:38:39.764Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"6f75934c-9822-4b02-99df-d400bba577f2","rayCluster":"raycluster","podIPs":["10.134.0.20","10.133.0.35"]}
{"level":"info","ts":"2026-06-05T09:38:41.724Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"7b355ac8-9ef4-4948-aabd-a75ec935df6d"}
{"level":"info","ts":"2026-06-05T09:38:41.724Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"7b355ac8-9ef4-4948-aabd-a75ec935df6d"}
{"level":"info","ts":"2026-06-05T09:38:41.724Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"7b355ac8-9ef4-4948-aabd-a75ec935df6d","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-05T09:38:41.724Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"7b355ac8-9ef4-4948-aabd-a75ec935df6d","Found 1 head Pod":"raycluster-head-vjd7d","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:38:41.724Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"7b355ac8-9ef4-4948-aabd-a75ec935df6d","head Pod":"raycluster-head-vjd7d","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-vjd7d. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:38:41.724Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"7b355ac8-9ef4-4948-aabd-a75ec935df6d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:38:41.724Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"7b355ac8-9ef4-4948-aabd-a75ec935df6d","worker Pod":"raycluster-small-group-worker-j9pbw","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-j9pbw. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:38:41.724Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"7b355ac8-9ef4-4948-aabd-a75ec935df6d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:38:41.724Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"7b355ac8-9ef4-4948-aabd-a75ec935df6d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:38:41.724Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"7b355ac8-9ef4-4948-aabd-a75ec935df6d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:38:41.724Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"7b355ac8-9ef4-4948-aabd-a75ec935df6d","podName":"raycluster-head-vjd7d","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:38:41.724Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"7b355ac8-9ef4-4948-aabd-a75ec935df6d","podName":"raycluster-head-vjd7d","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:38:41.724Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"7b355ac8-9ef4-4948-aabd-a75ec935df6d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:38:41.724Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"7b355ac8-9ef4-4948-aabd-a75ec935df6d","seconds":300}
{"level":"info","ts":"2026-06-05T09:38:44.719Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"a310f646-bc5a-4bc6-b3c8-7ec89c3c660a"}
{"level":"info","ts":"2026-06-05T09:38:44.719Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"a310f646-bc5a-4bc6-b3c8-7ec89c3c660a"}
{"level":"info","ts":"2026-06-05T09:38:44.719Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"a310f646-bc5a-4bc6-b3c8-7ec89c3c660a","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-05T09:38:44.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"a310f646-bc5a-4bc6-b3c8-7ec89c3c660a","Found 1 head Pod":"raycluster-head-vjd7d","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:38:44.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"a310f646-bc5a-4bc6-b3c8-7ec89c3c660a","head Pod":"raycluster-head-vjd7d","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-vjd7d. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:38:44.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"a310f646-bc5a-4bc6-b3c8-7ec89c3c660a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:38:44.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"a310f646-bc5a-4bc6-b3c8-7ec89c3c660a","worker Pod":"raycluster-small-group-worker-j9pbw","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-j9pbw. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:38:44.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"a310f646-bc5a-4bc6-b3c8-7ec89c3c660a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:38:44.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"a310f646-bc5a-4bc6-b3c8-7ec89c3c660a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:38:44.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"a310f646-bc5a-4bc6-b3c8-7ec89c3c660a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:38:44.719Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"a310f646-bc5a-4bc6-b3c8-7ec89c3c660a","podName":"raycluster-head-vjd7d","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:38:44.719Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"a310f646-bc5a-4bc6-b3c8-7ec89c3c660a","podName":"raycluster-head-vjd7d","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:38:44.719Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"a310f646-bc5a-4bc6-b3c8-7ec89c3c660a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:38:44.719Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"a310f646-bc5a-4bc6-b3c8-7ec89c3c660a","seconds":300}
{"level":"info","ts":"2026-06-05T09:38:49.949Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"3ebcea23-9833-4ed8-9160-9f63a09fd82f"}
{"level":"info","ts":"2026-06-05T09:38:49.949Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"3ebcea23-9833-4ed8-9160-9f63a09fd82f"}
{"level":"info","ts":"2026-06-05T09:38:49.949Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"3ebcea23-9833-4ed8-9160-9f63a09fd82f","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-05T09:38:49.949Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"3ebcea23-9833-4ed8-9160-9f63a09fd82f","Found 1 head Pod":"raycluster-head-vjd7d","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:38:49.949Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"3ebcea23-9833-4ed8-9160-9f63a09fd82f","head Pod":"raycluster-head-vjd7d","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-vjd7d. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:38:49.949Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"3ebcea23-9833-4ed8-9160-9f63a09fd82f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:38:49.949Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"3ebcea23-9833-4ed8-9160-9f63a09fd82f","worker Pod":"raycluster-small-group-worker-j9pbw","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-j9pbw. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:38:49.949Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"3ebcea23-9833-4ed8-9160-9f63a09fd82f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:38:49.949Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"3ebcea23-9833-4ed8-9160-9f63a09fd82f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:38:49.949Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"3ebcea23-9833-4ed8-9160-9f63a09fd82f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:38:49.949Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"3ebcea23-9833-4ed8-9160-9f63a09fd82f","podName":"raycluster-small-group-worker-j9pbw","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:38:49.949Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"3ebcea23-9833-4ed8-9160-9f63a09fd82f","podName":"raycluster-small-group-worker-j9pbw","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:38:49.949Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"3ebcea23-9833-4ed8-9160-9f63a09fd82f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:38:49.949Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"3ebcea23-9833-4ed8-9160-9f63a09fd82f","seconds":300}
{"level":"info","ts":"2026-06-05T09:38:50.952Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"1690b53f-bd6f-4aa2-b4a3-93bd8222bafc"}
{"level":"info","ts":"2026-06-05T09:38:50.952Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"1690b53f-bd6f-4aa2-b4a3-93bd8222bafc"}
{"level":"info","ts":"2026-06-05T09:38:50.952Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"1690b53f-bd6f-4aa2-b4a3-93bd8222bafc","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-05T09:38:50.953Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"1690b53f-bd6f-4aa2-b4a3-93bd8222bafc","Found 1 head Pod":"raycluster-head-vjd7d","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:38:50.953Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"1690b53f-bd6f-4aa2-b4a3-93bd8222bafc","head Pod":"raycluster-head-vjd7d","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-vjd7d. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:38:50.953Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"1690b53f-bd6f-4aa2-b4a3-93bd8222bafc","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:38:50.953Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"1690b53f-bd6f-4aa2-b4a3-93bd8222bafc","worker Pod":"raycluster-small-group-worker-j9pbw","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-j9pbw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:38:50.953Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"1690b53f-bd6f-4aa2-b4a3-93bd8222bafc","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:38:50.953Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"1690b53f-bd6f-4aa2-b4a3-93bd8222bafc","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:38:50.953Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"1690b53f-bd6f-4aa2-b4a3-93bd8222bafc","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:38:50.953Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"1690b53f-bd6f-4aa2-b4a3-93bd8222bafc","podName":"raycluster-head-vjd7d","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:38:50.953Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"1690b53f-bd6f-4aa2-b4a3-93bd8222bafc","podName":"raycluster-head-vjd7d","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:38:50.953Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"1690b53f-bd6f-4aa2-b4a3-93bd8222bafc","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-05T09:38:50.953Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"1690b53f-bd6f-4aa2-b4a3-93bd8222bafc","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:38:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.20","serviceIP":"10.134.0.20","podName":"raycluster-head-vjd7d","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:38:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.20","serviceIP":"10.134.0.20","podName":"raycluster-head-vjd7d","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:38:50.964Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"41d32366-05a9-4322-bf76-a9a2f3f1cd1d","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:38:50.964Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"1db91288-ef00-48c5-b479-755e827216af","cluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:38:50.964Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"41d32366-05a9-4322-bf76-a9a2f3f1cd1d","pod":"raycluster-head-vjd7d","ip":"10.134.0.20"}
{"level":"info","ts":"2026-06-05T09:38:50.964Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"41d32366-05a9-4322-bf76-a9a2f3f1cd1d","pod":"raycluster-small-group-worker-j9pbw","ip":"10.133.0.35"}
{"level":"info","ts":"2026-06-05T09:38:50.964Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"41d32366-05a9-4322-bf76-a9a2f3f1cd1d","rayCluster":"raycluster","podIPs":["10.134.0.20","10.133.0.35"]}
{"level":"info","ts":"2026-06-05T09:38:50.979Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"1db91288-ef00-48c5-b479-755e827216af","name":"raycluster-head"}
{"level":"info","ts":"2026-06-05T09:38:50.990Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"1db91288-ef00-48c5-b479-755e827216af","name":"raycluster-workers"}
{"level":"info","ts":"2026-06-05T09:38:50.990Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"1db91288-ef00-48c5-b479-755e827216af","cluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:38:52.964Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"8ba76cb4-7da5-4121-8340-b719aa7c08e1"}
{"level":"info","ts":"2026-06-05T09:38:52.964Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"8ba76cb4-7da5-4121-8340-b719aa7c08e1"}
{"level":"info","ts":"2026-06-05T09:38:52.964Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"8ba76cb4-7da5-4121-8340-b719aa7c08e1","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-05T09:38:52.964Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"8ba76cb4-7da5-4121-8340-b719aa7c08e1","Found 1 head Pod":"raycluster-head-vjd7d","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:38:52.964Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"8ba76cb4-7da5-4121-8340-b719aa7c08e1","head Pod":"raycluster-head-vjd7d","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-vjd7d. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:38:52.964Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"8ba76cb4-7da5-4121-8340-b719aa7c08e1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:38:52.964Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"8ba76cb4-7da5-4121-8340-b719aa7c08e1","worker Pod":"raycluster-small-group-worker-j9pbw","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-j9pbw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:38:52.964Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"8ba76cb4-7da5-4121-8340-b719aa7c08e1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:38:52.964Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"8ba76cb4-7da5-4121-8340-b719aa7c08e1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:38:52.964Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"8ba76cb4-7da5-4121-8340-b719aa7c08e1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:38:52.964Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"8ba76cb4-7da5-4121-8340-b719aa7c08e1","podName":"raycluster-head-vjd7d","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:38:52.964Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"8ba76cb4-7da5-4121-8340-b719aa7c08e1","podName":"raycluster-head-vjd7d","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:38:52.965Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"8ba76cb4-7da5-4121-8340-b719aa7c08e1","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:38:52.965Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"8ba76cb4-7da5-4121-8340-b719aa7c08e1","seconds":300}
{"level":"info","ts":"2026-06-05T09:38:53.793Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"ead34b3f-2c8b-44d0-a96c-c69f97223071"}
{"level":"info","ts":"2026-06-05T09:38:53.793Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"ead34b3f-2c8b-44d0-a96c-c69f97223071"}
{"level":"info","ts":"2026-06-05T09:38:53.793Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"ead34b3f-2c8b-44d0-a96c-c69f97223071","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-05T09:38:53.793Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"ead34b3f-2c8b-44d0-a96c-c69f97223071","Found 1 head Pod":"raycluster-head-vjd7d","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:38:53.793Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"ead34b3f-2c8b-44d0-a96c-c69f97223071","head Pod":"raycluster-head-vjd7d","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-vjd7d. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:38:53.793Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"ead34b3f-2c8b-44d0-a96c-c69f97223071","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:38:53.794Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"ead34b3f-2c8b-44d0-a96c-c69f97223071","worker Pod":"raycluster-small-group-worker-j9pbw","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-j9pbw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:38:53.794Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"ead34b3f-2c8b-44d0-a96c-c69f97223071","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:38:53.794Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"ead34b3f-2c8b-44d0-a96c-c69f97223071","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:38:53.794Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"ead34b3f-2c8b-44d0-a96c-c69f97223071","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:38:53.794Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"ead34b3f-2c8b-44d0-a96c-c69f97223071","podName":"raycluster-small-group-worker-j9pbw","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:35:09Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-05T09:38:53.794Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"ead34b3f-2c8b-44d0-a96c-c69f97223071","podName":"raycluster-small-group-worker-j9pbw","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:35:09Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-05T09:38:53.794Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"ead34b3f-2c8b-44d0-a96c-c69f97223071","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-05T09:38:53.794Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"ead34b3f-2c8b-44d0-a96c-c69f97223071","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:38:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.20","serviceIP":"10.134.0.20","podName":"raycluster-head-vjd7d","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:38:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.20","serviceIP":"10.134.0.20","podName":"raycluster-head-vjd7d","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:38:53.806Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"a93f39e7-289f-47d8-a712-a725510a255b","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:38:53.806Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"8b66f7c7-06f7-4fdb-aa76-410487446e2f","cluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:38:53.806Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"a93f39e7-289f-47d8-a712-a725510a255b","pod":"raycluster-head-vjd7d","ip":"10.134.0.20"}
{"level":"info","ts":"2026-06-05T09:38:53.806Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"a93f39e7-289f-47d8-a712-a725510a255b","pod":"raycluster-small-group-worker-j9pbw","ip":"10.133.0.35"}
{"level":"info","ts":"2026-06-05T09:38:53.806Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"a93f39e7-289f-47d8-a712-a725510a255b","rayCluster":"raycluster","podIPs":["10.134.0.20","10.133.0.35"]}
{"level":"info","ts":"2026-06-05T09:38:53.818Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"8b66f7c7-06f7-4fdb-aa76-410487446e2f","name":"raycluster-head"}
{"level":"info","ts":"2026-06-05T09:38:53.835Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"8b66f7c7-06f7-4fdb-aa76-410487446e2f","name":"raycluster-workers"}
{"level":"info","ts":"2026-06-05T09:38:53.835Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"8b66f7c7-06f7-4fdb-aa76-410487446e2f","cluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:38:55.806Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"72aea164-5230-49cf-9df7-5850909aaec4"}
{"level":"info","ts":"2026-06-05T09:38:55.806Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"72aea164-5230-49cf-9df7-5850909aaec4"}
{"level":"info","ts":"2026-06-05T09:38:55.806Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"72aea164-5230-49cf-9df7-5850909aaec4","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-05T09:38:55.807Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"72aea164-5230-49cf-9df7-5850909aaec4","Found 1 head Pod":"raycluster-head-vjd7d","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:38:55.807Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"72aea164-5230-49cf-9df7-5850909aaec4","head Pod":"raycluster-head-vjd7d","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-vjd7d. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:38:55.807Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"72aea164-5230-49cf-9df7-5850909aaec4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:38:55.807Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"72aea164-5230-49cf-9df7-5850909aaec4","worker Pod":"raycluster-small-group-worker-j9pbw","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-j9pbw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:38:55.807Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"72aea164-5230-49cf-9df7-5850909aaec4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:38:55.807Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"72aea164-5230-49cf-9df7-5850909aaec4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:38:55.807Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"72aea164-5230-49cf-9df7-5850909aaec4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:38:55.807Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"72aea164-5230-49cf-9df7-5850909aaec4","podName":"raycluster-head-vjd7d","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:35:09Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:38:55.807Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"72aea164-5230-49cf-9df7-5850909aaec4","podName":"raycluster-head-vjd7d","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:35:09Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:38:55.807Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"72aea164-5230-49cf-9df7-5850909aaec4","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:38:55.807Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"72aea164-5230-49cf-9df7-5850909aaec4","seconds":300}
{"level":"info","ts":"2026-06-05T09:38:59.829Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"8b69d464-8035-4a1a-a0cb-d3f1742d51e4"}
{"level":"info","ts":"2026-06-05T09:38:59.829Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"8b69d464-8035-4a1a-a0cb-d3f1742d51e4"}
{"level":"info","ts":"2026-06-05T09:38:59.829Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"8b69d464-8035-4a1a-a0cb-d3f1742d51e4","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-05T09:38:59.829Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"8b69d464-8035-4a1a-a0cb-d3f1742d51e4","Found 1 head Pod":"raycluster-head-vjd7d","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:38:59.829Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"8b69d464-8035-4a1a-a0cb-d3f1742d51e4","head Pod":"raycluster-head-vjd7d","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-vjd7d. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:38:59.829Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"8b69d464-8035-4a1a-a0cb-d3f1742d51e4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:38:59.829Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"8b69d464-8035-4a1a-a0cb-d3f1742d51e4","worker Pod":"raycluster-small-group-worker-j9pbw","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-j9pbw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:38:59.829Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"8b69d464-8035-4a1a-a0cb-d3f1742d51e4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:38:59.829Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"8b69d464-8035-4a1a-a0cb-d3f1742d51e4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:38:59.829Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"8b69d464-8035-4a1a-a0cb-d3f1742d51e4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:38:59.829Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"8b69d464-8035-4a1a-a0cb-d3f1742d51e4","podName":"raycluster-small-group-worker-j9pbw","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:35:09Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-05T09:38:59.829Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"8b69d464-8035-4a1a-a0cb-d3f1742d51e4","podName":"raycluster-small-group-worker-j9pbw","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:35:09Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-05T09:38:59.829Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"8b69d464-8035-4a1a-a0cb-d3f1742d51e4","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:38:59Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-05T09:38:59.829Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"8b69d464-8035-4a1a-a0cb-d3f1742d51e4","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:38:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.20","serviceIP":"10.134.0.20","podName":"raycluster-head-vjd7d","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:38:59Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.20","serviceIP":"10.134.0.20","podName":"raycluster-head-vjd7d","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:38:59Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:38:59.843Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"a7338fe5-7d12-472a-9ad3-d9e2d39bc58c","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:38:59.843Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"62ba03b2-8658-45f5-aab1-5d386d15e7d1","cluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:38:59.843Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"a7338fe5-7d12-472a-9ad3-d9e2d39bc58c","pod":"raycluster-head-vjd7d","ip":"10.134.0.20"}
{"level":"info","ts":"2026-06-05T09:38:59.843Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"a7338fe5-7d12-472a-9ad3-d9e2d39bc58c","pod":"raycluster-small-group-worker-j9pbw","ip":"10.133.0.35"}
{"level":"info","ts":"2026-06-05T09:38:59.843Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"a7338fe5-7d12-472a-9ad3-d9e2d39bc58c","rayCluster":"raycluster","podIPs":["10.134.0.20","10.133.0.35"]}
{"level":"info","ts":"2026-06-05T09:38:59.914Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"62ba03b2-8658-45f5-aab1-5d386d15e7d1","name":"raycluster-head"}
{"level":"info","ts":"2026-06-05T09:38:59.926Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"62ba03b2-8658-45f5-aab1-5d386d15e7d1","name":"raycluster-workers"}
{"level":"info","ts":"2026-06-05T09:38:59.926Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"62ba03b2-8658-45f5-aab1-5d386d15e7d1","cluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:39:01.844Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"51824d59-5eae-469d-91b3-d4a714c934ec"}
{"level":"info","ts":"2026-06-05T09:39:01.844Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"51824d59-5eae-469d-91b3-d4a714c934ec"}
{"level":"info","ts":"2026-06-05T09:39:01.844Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"51824d59-5eae-469d-91b3-d4a714c934ec","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:01.844Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"51824d59-5eae-469d-91b3-d4a714c934ec","Found 1 head Pod":"raycluster-head-vjd7d","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:39:01.844Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"51824d59-5eae-469d-91b3-d4a714c934ec","head Pod":"raycluster-head-vjd7d","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-vjd7d. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:01.844Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"51824d59-5eae-469d-91b3-d4a714c934ec","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:39:01.844Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"51824d59-5eae-469d-91b3-d4a714c934ec","worker Pod":"raycluster-small-group-worker-j9pbw","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-j9pbw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:01.844Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"51824d59-5eae-469d-91b3-d4a714c934ec","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:39:01.844Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"51824d59-5eae-469d-91b3-d4a714c934ec","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:39:01.844Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"51824d59-5eae-469d-91b3-d4a714c934ec","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:39:01.844Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"51824d59-5eae-469d-91b3-d4a714c934ec","podName":"raycluster-small-group-worker-j9pbw","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:35:09Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-05T09:39:01.844Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"51824d59-5eae-469d-91b3-d4a714c934ec","podName":"raycluster-small-group-worker-j9pbw","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:35:09Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-05T09:39:01.844Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"51824d59-5eae-469d-91b3-d4a714c934ec","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:39:01.844Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"51824d59-5eae-469d-91b3-d4a714c934ec","seconds":300}
{"level":"info","ts":"2026-06-05T09:39:01.984Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"076b29fc-ca7b-40f1-a427-091d4724ae18"}
{"level":"info","ts":"2026-06-05T09:39:01.984Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"076b29fc-ca7b-40f1-a427-091d4724ae18"}
{"level":"info","ts":"2026-06-05T09:39:01.984Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"076b29fc-ca7b-40f1-a427-091d4724ae18","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:01.984Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"076b29fc-ca7b-40f1-a427-091d4724ae18","Found 1 head Pod":"raycluster-head-vjd7d","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:39:01.984Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"076b29fc-ca7b-40f1-a427-091d4724ae18","head Pod":"raycluster-head-vjd7d","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-vjd7d. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:01.984Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"076b29fc-ca7b-40f1-a427-091d4724ae18","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:39:01.984Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"076b29fc-ca7b-40f1-a427-091d4724ae18","worker Pod":"raycluster-small-group-worker-j9pbw","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-j9pbw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:01.984Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"076b29fc-ca7b-40f1-a427-091d4724ae18","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:39:01.984Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"076b29fc-ca7b-40f1-a427-091d4724ae18","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:39:01.984Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"076b29fc-ca7b-40f1-a427-091d4724ae18","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:39:01.984Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"076b29fc-ca7b-40f1-a427-091d4724ae18","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-06-05T09:39:01.984Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"076b29fc-ca7b-40f1-a427-091d4724ae18","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:38:59Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.20","serviceIP":"10.134.0.20","podName":"raycluster-head-vjd7d","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:38:59Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:01Z","stateTransitionTimes":{"ready":"2026-06-05T09:39:01Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.20","serviceIP":"10.134.0.20","podName":"raycluster-head-vjd7d","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:38:59Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:39:01Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:39:01.998Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"ab87728f-fcf2-418a-b664-95e7be88b243","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:39:01.998Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"322145d4-d76f-4a5a-ab87-a75f41b338ec","cluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:39:01.998Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"ab87728f-fcf2-418a-b664-95e7be88b243","pod":"raycluster-head-vjd7d","ip":"10.134.0.20"}
{"level":"info","ts":"2026-06-05T09:39:01.998Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"ab87728f-fcf2-418a-b664-95e7be88b243","pod":"raycluster-small-group-worker-j9pbw","ip":"10.133.0.35"}
{"level":"info","ts":"2026-06-05T09:39:01.998Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"ab87728f-fcf2-418a-b664-95e7be88b243","rayCluster":"raycluster","podIPs":["10.134.0.20","10.133.0.35"]}
{"level":"info","ts":"2026-06-05T09:39:02.013Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"322145d4-d76f-4a5a-ab87-a75f41b338ec","name":"raycluster-head"}
{"level":"info","ts":"2026-06-05T09:39:02.025Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"322145d4-d76f-4a5a-ab87-a75f41b338ec","name":"raycluster-workers"}
{"level":"info","ts":"2026-06-05T09:39:02.025Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"322145d4-d76f-4a5a-ab87-a75f41b338ec","cluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:39:02.726Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"c9cc1139-3741-4938-a015-e0e7da66c073","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:02.726Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"c9cc1139-3741-4938-a015-e0e7da66c073","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-05T09:39:02.738Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"c9cc1139-3741-4938-a015-e0e7da66c073"}
{"level":"info","ts":"2026-06-05T09:39:02.738Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"c9cc1139-3741-4938-a015-e0e7da66c073","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-05T09:39:02.738Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"c9cc1139-3741-4938-a015-e0e7da66c073","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5mm55","rayClusterName":"counter-85b5h","jobDeploymentStatus":"Initializing","startTime":"2026-06-05T09:39:02Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:39:02.738Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"c9cc1139-3741-4938-a015-e0e7da66c073","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-05T09:39:02.749Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"0de13752-5ffb-41ef-a528-0ee8aad52c31","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:02.749Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"0de13752-5ffb-41ef-a528-0ee8aad52c31","name":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:02.749Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"0de13752-5ffb-41ef-a528-0ee8aad52c31","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:02.749Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"0de13752-5ffb-41ef-a528-0ee8aad52c31","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:02.768Z","logger":"raycluster-resource","msg":"default","name":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:02.769Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-85b5h","namespace":"test-ns-l4cdh"}
{"level":"info","ts":"2026-06-05T09:39:02.778Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"0de13752-5ffb-41ef-a528-0ee8aad52c31","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:02.778Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"f836f7fc-6521-4de7-b2b4-d1050c43c063","cluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:02.778Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"0de13752-5ffb-41ef-a528-0ee8aad52c31"}
{"level":"info","ts":"2026-06-05T09:39:02.778Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"0de13752-5ffb-41ef-a528-0ee8aad52c31","RayCluster":"counter-85b5h","State":""}
{"level":"info","ts":"2026-06-05T09:39:02.778Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"5e1665e7-6a35-4ec9-9ff4-9f10fdcebb4c"}
{"level":"info","ts":"2026-06-05T09:39:02.778Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"5e1665e7-6a35-4ec9-9ff4-9f10fdcebb4c"}
{"level":"info","ts":"2026-06-05T09:39:02.778Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"92a59a9c-4dfd-47ac-8808-063f647220e1","namespacedName":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:02.778Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"2c43769f-57d6-4135-b25f-a19f2802bb6e","rayCluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:02.778Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"2c43769f-57d6-4135-b25f-a19f2802bb6e","rayCluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:02.778Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"92a59a9c-4dfd-47ac-8808-063f647220e1","mode":"IntegratedOAuth","cluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:02.778Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"92a59a9c-4dfd-47ac-8808-063f647220e1","cluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:02.778Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"92a59a9c-4dfd-47ac-8808-063f647220e1","cluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:02.778Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"dc06ae67-733e-416f-b5ed-f1e99d9c4889","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:02.779Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"dc06ae67-733e-416f-b5ed-f1e99d9c4889","name":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:02.779Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"dc06ae67-733e-416f-b5ed-f1e99d9c4889","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:02.779Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"dc06ae67-733e-416f-b5ed-f1e99d9c4889"}
{"level":"info","ts":"2026-06-05T09:39:02.779Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"dc06ae67-733e-416f-b5ed-f1e99d9c4889","RayCluster":"counter-85b5h","State":""}
{"level":"info","ts":"2026-06-05T09:39:02.785Z","logger":"raycluster-resource","msg":"default","name":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:02.785Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-85b5h","namespace":"test-ns-l4cdh"}
{"level":"info","ts":"2026-06-05T09:39:02.787Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"5e1665e7-6a35-4ec9-9ff4-9f10fdcebb4c","name":"counter-85b5h-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:02.787Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"5e1665e7-6a35-4ec9-9ff4-9f10fdcebb4c","error":"head certificate secret ray-head-secret-counter-85b5h not found"}
{"level":"error","ts":"2026-06-05T09:39:02.787Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"5e1665e7-6a35-4ec9-9ff4-9f10fdcebb4c","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-85b5h not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:02.787Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"5e1665e7-6a35-4ec9-9ff4-9f10fdcebb4c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-85b5h not found"}
{"level":"info","ts":"2026-06-05T09:39:02.787Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"5e1665e7-6a35-4ec9-9ff4-9f10fdcebb4c","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"counter-85b5h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:02.787Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"5e1665e7-6a35-4ec9-9ff4-9f10fdcebb4c","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"counter-85b5h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:02.787Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"5e1665e7-6a35-4ec9-9ff4-9f10fdcebb4c","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"counter-85b5h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:02.787Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"5e1665e7-6a35-4ec9-9ff4-9f10fdcebb4c","oldEndpoints":null,"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{},"newHead":{"serviceName":"counter-85b5h-head-svc"}}
{"level":"info","ts":"2026-06-05T09:39:02.787Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"5e1665e7-6a35-4ec9-9ff4-9f10fdcebb4c","name":"counter-85b5h","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-85b5h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterSuspending","message":""}],"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:39:02.806Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"5e1665e7-6a35-4ec9-9ff4-9f10fdcebb4c","name":"counter-85b5h","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-85b5h\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-l4cdh","name":"counter-85b5h"}}
{"level":"info","ts":"2026-06-05T09:39:02.806Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"5e1665e7-6a35-4ec9-9ff4-9f10fdcebb4c"}
{"level":"info","ts":"2026-06-05T09:39:02.806Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"f836f7fc-6521-4de7-b2b4-d1050c43c063","cluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:02.806Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"2c43769f-57d6-4135-b25f-a19f2802bb6e","rayCluster":"counter-85b5h"}
{"level":"error","ts":"2026-06-05T09:39:02.806Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"5e1665e7-6a35-4ec9-9ff4-9f10fdcebb4c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-85b5h not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:02.806Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"2c43769f-57d6-4135-b25f-a19f2802bb6e","rayCluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:02.806Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"b29f5497-087f-4fce-a856-69feda364dcc","cluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:02.806Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"928d3226-7a3b-4ae3-8b3e-0d7bb6e0d7f2"}
{"level":"info","ts":"2026-06-05T09:39:02.806Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"928d3226-7a3b-4ae3-8b3e-0d7bb6e0d7f2"}
{"level":"info","ts":"2026-06-05T09:39:02.806Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"928d3226-7a3b-4ae3-8b3e-0d7bb6e0d7f2","1 head service found":"counter-85b5h-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:02.806Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"928d3226-7a3b-4ae3-8b3e-0d7bb6e0d7f2","error":"head certificate secret ray-head-secret-counter-85b5h not found"}
{"level":"error","ts":"2026-06-05T09:39:02.806Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"928d3226-7a3b-4ae3-8b3e-0d7bb6e0d7f2","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-85b5h not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:02.806Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"928d3226-7a3b-4ae3-8b3e-0d7bb6e0d7f2","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-85b5h not found"}
{"level":"info","ts":"2026-06-05T09:39:02.806Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"928d3226-7a3b-4ae3-8b3e-0d7bb6e0d7f2","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"counter-85b5h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:02.806Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"928d3226-7a3b-4ae3-8b3e-0d7bb6e0d7f2","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"counter-85b5h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:02.806Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"928d3226-7a3b-4ae3-8b3e-0d7bb6e0d7f2","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"counter-85b5h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:02.806Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"0945dca5-7a46-4361-a6b1-912f1d65ae7f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:02.806Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"0945dca5-7a46-4361-a6b1-912f1d65ae7f","name":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:02.806Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"928d3226-7a3b-4ae3-8b3e-0d7bb6e0d7f2","oldEndpoints":null,"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{},"newHead":{"serviceName":"counter-85b5h-head-svc"}}
{"level":"info","ts":"2026-06-05T09:39:02.806Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"0945dca5-7a46-4361-a6b1-912f1d65ae7f","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:02.806Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"928d3226-7a3b-4ae3-8b3e-0d7bb6e0d7f2","name":"counter-85b5h","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-85b5h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterSuspending","message":""}],"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:39:02.806Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"0945dca5-7a46-4361-a6b1-912f1d65ae7f"}
{"level":"info","ts":"2026-06-05T09:39:02.806Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"0945dca5-7a46-4361-a6b1-912f1d65ae7f","RayCluster":"counter-85b5h","State":""}
{"level":"info","ts":"2026-06-05T09:39:02.812Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"92a59a9c-4dfd-47ac-8808-063f647220e1","name":"counter-85b5h-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:39:02.816Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"2c43769f-57d6-4135-b25f-a19f2802bb6e"}
{"level":"info","ts":"2026-06-05T09:39:02.816Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"928d3226-7a3b-4ae3-8b3e-0d7bb6e0d7f2","name":"counter-85b5h","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-85b5h\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-l4cdh","name":"counter-85b5h"}}
{"level":"info","ts":"2026-06-05T09:39:02.816Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"928d3226-7a3b-4ae3-8b3e-0d7bb6e0d7f2"}
{"level":"error","ts":"2026-06-05T09:39:02.816Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"928d3226-7a3b-4ae3-8b3e-0d7bb6e0d7f2","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-85b5h not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:02.817Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"bc0889fa-48c6-41ab-8355-219a836e03a7"}
{"level":"info","ts":"2026-06-05T09:39:02.817Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"bc0889fa-48c6-41ab-8355-219a836e03a7"}
{"level":"info","ts":"2026-06-05T09:39:02.817Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"bc0889fa-48c6-41ab-8355-219a836e03a7","1 head service found":"counter-85b5h-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:02.817Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"bc0889fa-48c6-41ab-8355-219a836e03a7","error":"head certificate secret ray-head-secret-counter-85b5h not found"}
{"level":"error","ts":"2026-06-05T09:39:02.817Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"bc0889fa-48c6-41ab-8355-219a836e03a7","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-85b5h not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:02.817Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"bc0889fa-48c6-41ab-8355-219a836e03a7","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-85b5h not found"}
{"level":"info","ts":"2026-06-05T09:39:02.817Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"bc0889fa-48c6-41ab-8355-219a836e03a7","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"counter-85b5h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:02.817Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"bc0889fa-48c6-41ab-8355-219a836e03a7","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"counter-85b5h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:02.817Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"bc0889fa-48c6-41ab-8355-219a836e03a7","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"counter-85b5h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:02.817Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"bc0889fa-48c6-41ab-8355-219a836e03a7","oldEndpoints":null,"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{},"newHead":{"serviceName":"counter-85b5h-head-svc"}}
{"level":"info","ts":"2026-06-05T09:39:02.817Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"bc0889fa-48c6-41ab-8355-219a836e03a7","name":"counter-85b5h","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-85b5h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterSuspending","message":""}],"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:39:02.817Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"2c43769f-57d6-4135-b25f-a19f2802bb6e","rayCluster":"counter-85b5h","secret":"ca-secret"}
{"level":"info","ts":"2026-06-05T09:39:02.817Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"2c43769f-57d6-4135-b25f-a19f2802bb6e","rayCluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:02.906Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"2c43769f-57d6-4135-b25f-a19f2802bb6e","rayCluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:02.906Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"92a59a9c-4dfd-47ac-8808-063f647220e1","name":"kuberay-gateway-access","namespace":"test-ns-l4cdh","operation":"created"}
{"level":"info","ts":"2026-06-05T09:39:02.906Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"bc0889fa-48c6-41ab-8355-219a836e03a7"}
{"level":"error","ts":"2026-06-05T09:39:02.906Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"bc0889fa-48c6-41ab-8355-219a836e03a7","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-85b5h not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:02.907Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"c769afd0-7b1f-45fd-b7ec-2508f7755b98"}
{"level":"info","ts":"2026-06-05T09:39:02.907Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"c769afd0-7b1f-45fd-b7ec-2508f7755b98"}
{"level":"info","ts":"2026-06-05T09:39:02.907Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"c769afd0-7b1f-45fd-b7ec-2508f7755b98","1 head service found":"counter-85b5h-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:02.907Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"c769afd0-7b1f-45fd-b7ec-2508f7755b98","error":"head certificate secret ray-head-secret-counter-85b5h not found"}
{"level":"error","ts":"2026-06-05T09:39:02.907Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"c769afd0-7b1f-45fd-b7ec-2508f7755b98","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-85b5h not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:02.907Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"c769afd0-7b1f-45fd-b7ec-2508f7755b98","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-85b5h not found"}
{"level":"info","ts":"2026-06-05T09:39:02.907Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"c769afd0-7b1f-45fd-b7ec-2508f7755b98","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"counter-85b5h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:02.907Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"c769afd0-7b1f-45fd-b7ec-2508f7755b98","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"counter-85b5h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:02.907Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"c769afd0-7b1f-45fd-b7ec-2508f7755b98","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"counter-85b5h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:02.907Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"c769afd0-7b1f-45fd-b7ec-2508f7755b98","oldEndpoints":null,"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{},"newHead":{"serviceName":"counter-85b5h-head-svc"}}
{"level":"info","ts":"2026-06-05T09:39:02.907Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"c769afd0-7b1f-45fd-b7ec-2508f7755b98","name":"counter-85b5h","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-85b5h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterSuspending","message":""}],"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:39:02.907Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"b1e9d377-0d62-460b-9106-96b1cb089bd9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:02.907Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"b1e9d377-0d62-460b-9106-96b1cb089bd9","name":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:02.907Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"b1e9d377-0d62-460b-9106-96b1cb089bd9","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:02.907Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"b1e9d377-0d62-460b-9106-96b1cb089bd9"}
{"level":"info","ts":"2026-06-05T09:39:02.907Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"b1e9d377-0d62-460b-9106-96b1cb089bd9","RayCluster":"counter-85b5h","State":""}
{"level":"info","ts":"2026-06-05T09:39:02.911Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"b29f5497-087f-4fce-a856-69feda364dcc","name":"counter-85b5h-head"}
{"level":"info","ts":"2026-06-05T09:39:02.913Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"c769afd0-7b1f-45fd-b7ec-2508f7755b98","name":"counter-85b5h","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-85b5h\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-l4cdh","name":"counter-85b5h"}}
{"level":"info","ts":"2026-06-05T09:39:02.913Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"c769afd0-7b1f-45fd-b7ec-2508f7755b98"}
{"level":"error","ts":"2026-06-05T09:39:02.913Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"c769afd0-7b1f-45fd-b7ec-2508f7755b98","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-85b5h not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:02.915Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"92a59a9c-4dfd-47ac-8808-063f647220e1","name":"test-ns-l4cdh-counter-85b5h","operation":"created"}
{"level":"info","ts":"2026-06-05T09:39:02.918Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"2c43769f-57d6-4135-b25f-a19f2802bb6e"}
{"level":"info","ts":"2026-06-05T09:39:02.918Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"2c43769f-57d6-4135-b25f-a19f2802bb6e","rayCluster":"counter-85b5h","podIPs":[]}
{"level":"info","ts":"2026-06-05T09:39:02.920Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"92a59a9c-4dfd-47ac-8808-063f647220e1","name":"kube-rbac-proxy-config-counter-85b5h","operation":"created"}
{"level":"info","ts":"2026-06-05T09:39:02.923Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"b29f5497-087f-4fce-a856-69feda364dcc","name":"counter-85b5h-workers"}
{"level":"info","ts":"2026-06-05T09:39:02.923Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"b29f5497-087f-4fce-a856-69feda364dcc","cluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:02.923Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"f4dd7f28-c03f-4352-a0e5-9127280b7f8b","cluster":"counter-85b5h"}
{"level":"error","ts":"2026-06-05T09:39:02.926Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"92a59a9c-4dfd-47ac-8808-063f647220e1","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-85b5h\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:02.926Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"92a59a9c-4dfd-47ac-8808-063f647220e1"}
{"level":"error","ts":"2026-06-05T09:39:02.926Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"92a59a9c-4dfd-47ac-8808-063f647220e1","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-85b5h\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:02.926Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"539f93d4-6f8d-41a2-a6d7-e6e23bbc4a0f","namespacedName":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:02.927Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"7c2d8fcb-566a-49e2-9398-8f5c7d61bbc4"}
{"level":"info","ts":"2026-06-05T09:39:02.927Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"7c2d8fcb-566a-49e2-9398-8f5c7d61bbc4"}
{"level":"info","ts":"2026-06-05T09:39:02.927Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"7c2d8fcb-566a-49e2-9398-8f5c7d61bbc4","1 head service found":"counter-85b5h-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:02.927Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"7c2d8fcb-566a-49e2-9398-8f5c7d61bbc4","error":"head certificate secret ray-head-secret-counter-85b5h not found"}
{"level":"error","ts":"2026-06-05T09:39:02.927Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"7c2d8fcb-566a-49e2-9398-8f5c7d61bbc4","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-85b5h not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:02.927Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"7c2d8fcb-566a-49e2-9398-8f5c7d61bbc4","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-85b5h not found"}
{"level":"info","ts":"2026-06-05T09:39:02.927Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"7c2d8fcb-566a-49e2-9398-8f5c7d61bbc4","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"counter-85b5h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:02.927Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"7c2d8fcb-566a-49e2-9398-8f5c7d61bbc4","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"counter-85b5h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:02.928Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"7c2d8fcb-566a-49e2-9398-8f5c7d61bbc4","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"counter-85b5h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:02.928Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"7c2d8fcb-566a-49e2-9398-8f5c7d61bbc4"}
{"level":"error","ts":"2026-06-05T09:39:02.928Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"7c2d8fcb-566a-49e2-9398-8f5c7d61bbc4","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-85b5h not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:03.005Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"539f93d4-6f8d-41a2-a6d7-e6e23bbc4a0f","mode":"IntegratedOAuth","cluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:03.005Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"539f93d4-6f8d-41a2-a6d7-e6e23bbc4a0f","cluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:03.006Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"2c43769f-57d6-4135-b25f-a19f2802bb6e"}
{"level":"info","ts":"2026-06-05T09:39:03.006Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"2c43769f-57d6-4135-b25f-a19f2802bb6e","rayCluster":"counter-85b5h","podIPs":[]}
{"level":"error","ts":"2026-06-05T09:39:03.006Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"2c43769f-57d6-4135-b25f-a19f2802bb6e","error":"Certificate.cert-manager.io \"ray-worker-cert-counter-85b5h\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:03.006Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"2c43769f-57d6-4135-b25f-a19f2802bb6e"}
{"level":"error","ts":"2026-06-05T09:39:03.006Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"2c43769f-57d6-4135-b25f-a19f2802bb6e","error":"Certificate.cert-manager.io \"ray-worker-cert-counter-85b5h\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:03.006Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"f0383974-9885-4ac9-98d6-1bc049b49b57","rayCluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:03.006Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"f0383974-9885-4ac9-98d6-1bc049b49b57"}
{"level":"info","ts":"2026-06-05T09:39:03.008Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"53ff52de-5b19-4127-8011-944e03cd6bf0"}
{"level":"info","ts":"2026-06-05T09:39:03.008Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"53ff52de-5b19-4127-8011-944e03cd6bf0"}
{"level":"info","ts":"2026-06-05T09:39:03.008Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"53ff52de-5b19-4127-8011-944e03cd6bf0","1 head service found":"counter-85b5h-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:03.008Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"53ff52de-5b19-4127-8011-944e03cd6bf0","error":"head certificate secret ray-head-secret-counter-85b5h not found"}
{"level":"error","ts":"2026-06-05T09:39:03.008Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"53ff52de-5b19-4127-8011-944e03cd6bf0","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-85b5h not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:03.008Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"53ff52de-5b19-4127-8011-944e03cd6bf0","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-85b5h not found"}
{"level":"info","ts":"2026-06-05T09:39:03.008Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"53ff52de-5b19-4127-8011-944e03cd6bf0","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"counter-85b5h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:03.008Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"53ff52de-5b19-4127-8011-944e03cd6bf0","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"counter-85b5h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:03.008Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"53ff52de-5b19-4127-8011-944e03cd6bf0","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"counter-85b5h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:03.008Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"53ff52de-5b19-4127-8011-944e03cd6bf0"}
{"level":"error","ts":"2026-06-05T09:39:03.008Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"53ff52de-5b19-4127-8011-944e03cd6bf0","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-85b5h not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:03.011Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"f4dd7f28-c03f-4352-a0e5-9127280b7f8b","name":"counter-85b5h-head"}
{"level":"info","ts":"2026-06-05T09:39:03.011Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"4f0b8bbd-b4ee-4eb3-9234-77e5a9402c32","rayCluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:03.011Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"4f0b8bbd-b4ee-4eb3-9234-77e5a9402c32"}
{"level":"info","ts":"2026-06-05T09:39:03.018Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"539f93d4-6f8d-41a2-a6d7-e6e23bbc4a0f","cluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:03.018Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"c8ee980e-14f9-417d-ba1e-bad77436491b","namespacedName":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:03.018Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"f1488d9a-d625-44de-a664-aa4f31710ef2","rayCluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:03.018Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"c8ee980e-14f9-417d-ba1e-bad77436491b","mode":"IntegratedOAuth","cluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:03.018Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"c8ee980e-14f9-417d-ba1e-bad77436491b","cluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:03.018Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"f1488d9a-d625-44de-a664-aa4f31710ef2"}
{"level":"info","ts":"2026-06-05T09:39:03.019Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"283da337-3ef4-4037-bcc6-229f7f76e464","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:03.019Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"283da337-3ef4-4037-bcc6-229f7f76e464","name":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:03.019Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"283da337-3ef4-4037-bcc6-229f7f76e464","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:03.019Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"283da337-3ef4-4037-bcc6-229f7f76e464"}
{"level":"info","ts":"2026-06-05T09:39:03.019Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"283da337-3ef4-4037-bcc6-229f7f76e464","RayCluster":"counter-85b5h","State":""}
{"level":"info","ts":"2026-06-05T09:39:03.024Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"f4dd7f28-c03f-4352-a0e5-9127280b7f8b","name":"counter-85b5h-workers"}
{"level":"info","ts":"2026-06-05T09:39:03.024Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"f4dd7f28-c03f-4352-a0e5-9127280b7f8b","cluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:03.024Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"fa7898a8-7d4b-477f-b8d2-93e5bfb3c80d","cluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:03.106Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"c8ee980e-14f9-417d-ba1e-bad77436491b","cluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:03.111Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"fa7898a8-7d4b-477f-b8d2-93e5bfb3c80d","name":"counter-85b5h-head"}
{"level":"info","ts":"2026-06-05T09:39:03.121Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"fa7898a8-7d4b-477f-b8d2-93e5bfb3c80d","name":"counter-85b5h-workers"}
{"level":"info","ts":"2026-06-05T09:39:03.121Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"fa7898a8-7d4b-477f-b8d2-93e5bfb3c80d","cluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:03.169Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"5d8c1109-3f03-4ebe-83d6-6a3e6db1a7f5"}
{"level":"info","ts":"2026-06-05T09:39:03.169Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"5d8c1109-3f03-4ebe-83d6-6a3e6db1a7f5"}
{"level":"info","ts":"2026-06-05T09:39:03.169Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"5d8c1109-3f03-4ebe-83d6-6a3e6db1a7f5","1 head service found":"counter-85b5h-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:03.169Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"5d8c1109-3f03-4ebe-83d6-6a3e6db1a7f5","error":"head certificate secret ray-head-secret-counter-85b5h not found"}
{"level":"error","ts":"2026-06-05T09:39:03.169Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"5d8c1109-3f03-4ebe-83d6-6a3e6db1a7f5","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-85b5h not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:03.169Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"5d8c1109-3f03-4ebe-83d6-6a3e6db1a7f5","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-85b5h not found"}
{"level":"info","ts":"2026-06-05T09:39:03.169Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"5d8c1109-3f03-4ebe-83d6-6a3e6db1a7f5","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"counter-85b5h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:03.169Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"5d8c1109-3f03-4ebe-83d6-6a3e6db1a7f5","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"counter-85b5h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:03.169Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"5d8c1109-3f03-4ebe-83d6-6a3e6db1a7f5","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"counter-85b5h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:03.169Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"5d8c1109-3f03-4ebe-83d6-6a3e6db1a7f5"}
{"level":"error","ts":"2026-06-05T09:39:03.169Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"5d8c1109-3f03-4ebe-83d6-6a3e6db1a7f5","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-85b5h not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:03.490Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"9b17c22c-3cce-4ade-b03d-6079fcfcda0d"}
{"level":"info","ts":"2026-06-05T09:39:03.490Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"9b17c22c-3cce-4ade-b03d-6079fcfcda0d"}
{"level":"info","ts":"2026-06-05T09:39:03.490Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"9b17c22c-3cce-4ade-b03d-6079fcfcda0d","1 head service found":"counter-85b5h-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:03.490Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"9b17c22c-3cce-4ade-b03d-6079fcfcda0d","error":"head certificate secret ray-head-secret-counter-85b5h not found"}
{"level":"error","ts":"2026-06-05T09:39:03.490Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"9b17c22c-3cce-4ade-b03d-6079fcfcda0d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-85b5h not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:03.490Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"9b17c22c-3cce-4ade-b03d-6079fcfcda0d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-85b5h not found"}
{"level":"info","ts":"2026-06-05T09:39:03.490Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"9b17c22c-3cce-4ade-b03d-6079fcfcda0d","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"counter-85b5h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:03.490Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"9b17c22c-3cce-4ade-b03d-6079fcfcda0d","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"counter-85b5h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:03.490Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"9b17c22c-3cce-4ade-b03d-6079fcfcda0d","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"counter-85b5h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:03.490Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"9b17c22c-3cce-4ade-b03d-6079fcfcda0d"}
{"level":"error","ts":"2026-06-05T09:39:03.490Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"9b17c22c-3cce-4ade-b03d-6079fcfcda0d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-85b5h not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:03.999Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"80a4ff76-bf28-4985-ae2b-0312d98f2702"}
{"level":"info","ts":"2026-06-05T09:39:03.999Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"80a4ff76-bf28-4985-ae2b-0312d98f2702"}
{"level":"info","ts":"2026-06-05T09:39:03.999Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"80a4ff76-bf28-4985-ae2b-0312d98f2702","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:03.999Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"80a4ff76-bf28-4985-ae2b-0312d98f2702","Found 1 head Pod":"raycluster-head-vjd7d","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:39:03.999Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"80a4ff76-bf28-4985-ae2b-0312d98f2702","head Pod":"raycluster-head-vjd7d","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-vjd7d. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:03.999Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"80a4ff76-bf28-4985-ae2b-0312d98f2702","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:39:03.999Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"80a4ff76-bf28-4985-ae2b-0312d98f2702","worker Pod":"raycluster-small-group-worker-j9pbw","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-j9pbw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:03.999Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"80a4ff76-bf28-4985-ae2b-0312d98f2702","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:39:03.999Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"80a4ff76-bf28-4985-ae2b-0312d98f2702","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:39:03.999Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"80a4ff76-bf28-4985-ae2b-0312d98f2702","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:39:03.999Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"80a4ff76-bf28-4985-ae2b-0312d98f2702","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:39:03.999Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"80a4ff76-bf28-4985-ae2b-0312d98f2702","seconds":300}
{"level":"info","ts":"2026-06-05T09:39:04.130Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"38b7b640-07e1-4d3f-8c41-719ca676c12d"}
{"level":"info","ts":"2026-06-05T09:39:04.130Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"38b7b640-07e1-4d3f-8c41-719ca676c12d"}
{"level":"info","ts":"2026-06-05T09:39:04.130Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"38b7b640-07e1-4d3f-8c41-719ca676c12d","1 head service found":"counter-85b5h-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:04.130Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"38b7b640-07e1-4d3f-8c41-719ca676c12d","error":"head certificate secret ray-head-secret-counter-85b5h not found"}
{"level":"error","ts":"2026-06-05T09:39:04.130Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"38b7b640-07e1-4d3f-8c41-719ca676c12d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-85b5h not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:04.130Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"38b7b640-07e1-4d3f-8c41-719ca676c12d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-85b5h not found"}
{"level":"info","ts":"2026-06-05T09:39:04.130Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"38b7b640-07e1-4d3f-8c41-719ca676c12d","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"counter-85b5h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:04.130Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"38b7b640-07e1-4d3f-8c41-719ca676c12d","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"counter-85b5h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:04.130Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"38b7b640-07e1-4d3f-8c41-719ca676c12d","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"counter-85b5h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:04.131Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"38b7b640-07e1-4d3f-8c41-719ca676c12d"}
{"level":"error","ts":"2026-06-05T09:39:04.131Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"38b7b640-07e1-4d3f-8c41-719ca676c12d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-85b5h not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:05.411Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"8e237ba8-fd26-488c-9fca-e17a234e495d"}
{"level":"info","ts":"2026-06-05T09:39:05.411Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"8e237ba8-fd26-488c-9fca-e17a234e495d"}
{"level":"info","ts":"2026-06-05T09:39:05.411Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"8e237ba8-fd26-488c-9fca-e17a234e495d","1 head service found":"counter-85b5h-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:05.411Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"8e237ba8-fd26-488c-9fca-e17a234e495d"}
{"level":"info","ts":"2026-06-05T09:39:05.411Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"8e237ba8-fd26-488c-9fca-e17a234e495d","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:39:05.411Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"8e237ba8-fd26-488c-9fca-e17a234e495d"}
{"level":"info","ts":"2026-06-05T09:39:05.411Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"8e237ba8-fd26-488c-9fca-e17a234e495d","mode":"IntegratedOAuth","cluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:05.411Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"8e237ba8-fd26-488c-9fca-e17a234e495d","cluster":"counter-85b5h","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:39:05.411Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"8e237ba8-fd26-488c-9fca-e17a234e495d","cluster":"counter-85b5h","authType":"IntegratedOAuth","serviceAccount":"counter-85b5h-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-05T09:39:05.411Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"8e237ba8-fd26-488c-9fca-e17a234e495d","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"counter-85b5h","ray.io/group":"headgroup","ray.io/identifier":"counter-85b5h-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-05T09:39:05.411Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"8e237ba8-fd26-488c-9fca-e17a234e495d","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"counter-85b5h-head-svc.test-ns-l4cdh.svc.cluster.local","num-cpus":"4","num-gpus":"4","resources":"'{\"R1\": 4}'"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-05T09:39:05.411Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"8e237ba8-fd26-488c-9fca-e17a234e495d","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=counter-85b5h-head-svc.test-ns-l4cdh.svc.cluster.local  --num-cpus=4  --num-gpus=4  --resources='{\"R1\": 4}' "}
{"level":"info","ts":"2026-06-05T09:39:05.411Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"8e237ba8-fd26-488c-9fca-e17a234e495d","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=counter-85b5h-head-svc.test-ns-l4cdh.svc.cluster.local  --num-cpus=4  --num-gpus=4  --resources='{\"R1\": 4}' "}
{"level":"info","ts":"2026-06-05T09:39:05.411Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"8e237ba8-fd26-488c-9fca-e17a234e495d","enabled":true}
{"level":"info","ts":"2026-06-05T09:39:05.431Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"8e237ba8-fd26-488c-9fca-e17a234e495d","name":"counter-85b5h-head-tnx7c"}
{"level":"info","ts":"2026-06-05T09:39:05.431Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"8e237ba8-fd26-488c-9fca-e17a234e495d","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"counter-85b5h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:05.431Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"8e237ba8-fd26-488c-9fca-e17a234e495d","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"counter-85b5h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:05.431Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"8e237ba8-fd26-488c-9fca-e17a234e495d","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"counter-85b5h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:05.431Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"8e237ba8-fd26-488c-9fca-e17a234e495d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:39:05.431Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"8e237ba8-fd26-488c-9fca-e17a234e495d","seconds":300}
{"level":"info","ts":"2026-06-05T09:39:05.432Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"3c290c17-a4a2-4e27-b617-941c56360de9"}
{"level":"info","ts":"2026-06-05T09:39:05.432Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"3c290c17-a4a2-4e27-b617-941c56360de9"}
{"level":"info","ts":"2026-06-05T09:39:05.432Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"3c290c17-a4a2-4e27-b617-941c56360de9","1 head service found":"counter-85b5h-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:05.432Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"3c290c17-a4a2-4e27-b617-941c56360de9","Found 1 head Pod":"counter-85b5h-head-tnx7c","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:39:05.432Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"3c290c17-a4a2-4e27-b617-941c56360de9","head Pod":"counter-85b5h-head-tnx7c","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-85b5h-head-tnx7c. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:05.432Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"3c290c17-a4a2-4e27-b617-941c56360de9","podName":"counter-85b5h-head-tnx7c","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:39:05.432Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"3c290c17-a4a2-4e27-b617-941c56360de9","podName":"counter-85b5h-head-tnx7c","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:39:05.432Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"3c290c17-a4a2-4e27-b617-941c56360de9","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"counter-85b5h-head-svc"},"newHead":{"podName":"counter-85b5h-head-tnx7c","serviceName":"counter-85b5h-head-svc"}}
{"level":"info","ts":"2026-06-05T09:39:05.432Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"3c290c17-a4a2-4e27-b617-941c56360de9","name":"counter-85b5h","old status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-85b5h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-85b5h-head-tnx7c","serviceName":"counter-85b5h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:39:05.445Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"2752a103-a08a-4171-baeb-e57b35b341bf"}
{"level":"info","ts":"2026-06-05T09:39:05.445Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"2752a103-a08a-4171-baeb-e57b35b341bf"}
{"level":"info","ts":"2026-06-05T09:39:05.445Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"2752a103-a08a-4171-baeb-e57b35b341bf","1 head service found":"counter-85b5h-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:05.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"2752a103-a08a-4171-baeb-e57b35b341bf","Found 1 head Pod":"counter-85b5h-head-tnx7c","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:39:05.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"2752a103-a08a-4171-baeb-e57b35b341bf","head Pod":"counter-85b5h-head-tnx7c","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-85b5h-head-tnx7c. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:05.445Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"2752a103-a08a-4171-baeb-e57b35b341bf","podName":"counter-85b5h-head-tnx7c","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:39:05.445Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"2752a103-a08a-4171-baeb-e57b35b341bf","podName":"counter-85b5h-head-tnx7c","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:39:05.445Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"2752a103-a08a-4171-baeb-e57b35b341bf","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"counter-85b5h-head-svc"},"newHead":{"podName":"counter-85b5h-head-tnx7c","serviceName":"counter-85b5h-head-svc"}}
{"level":"info","ts":"2026-06-05T09:39:05.445Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"2752a103-a08a-4171-baeb-e57b35b341bf","name":"counter-85b5h","old status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-85b5h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-85b5h-head-tnx7c","serviceName":"counter-85b5h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:39:05.445Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"1aa49fbc-6541-4931-80a1-456a288dd5dc","rayCluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:05.445Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"88285706-209a-4ece-bbab-a2b3bb460d2e","cluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:05.446Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"1aa49fbc-6541-4931-80a1-456a288dd5dc","rayCluster":"counter-85b5h","podIPs":[]}
{"level":"info","ts":"2026-06-05T09:39:05.446Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"d99c8d92-99f1-4d99-9c6e-9cc60e2edbe7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:05.446Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"d99c8d92-99f1-4d99-9c6e-9cc60e2edbe7","name":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:05.446Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"d99c8d92-99f1-4d99-9c6e-9cc60e2edbe7","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:05.446Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"d99c8d92-99f1-4d99-9c6e-9cc60e2edbe7"}
{"level":"info","ts":"2026-06-05T09:39:05.446Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"d99c8d92-99f1-4d99-9c6e-9cc60e2edbe7","RayCluster":"counter-85b5h","State":""}
{"level":"info","ts":"2026-06-05T09:39:05.453Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"2752a103-a08a-4171-baeb-e57b35b341bf","name":"counter-85b5h","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-85b5h\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-l4cdh","name":"counter-85b5h"}}
{"level":"info","ts":"2026-06-05T09:39:05.453Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"2752a103-a08a-4171-baeb-e57b35b341bf"}
{"level":"error","ts":"2026-06-05T09:39:05.453Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"2752a103-a08a-4171-baeb-e57b35b341bf","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-85b5h\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:05.457Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"49f43a87-0c8b-4793-ae83-32aa5034dd3b"}
{"level":"info","ts":"2026-06-05T09:39:05.457Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"49f43a87-0c8b-4793-ae83-32aa5034dd3b"}
{"level":"info","ts":"2026-06-05T09:39:05.457Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"49f43a87-0c8b-4793-ae83-32aa5034dd3b","1 head service found":"counter-85b5h-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:05.457Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"49f43a87-0c8b-4793-ae83-32aa5034dd3b","Found 1 head Pod":"counter-85b5h-head-tnx7c","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:39:05.457Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"49f43a87-0c8b-4793-ae83-32aa5034dd3b","head Pod":"counter-85b5h-head-tnx7c","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-85b5h-head-tnx7c. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:05.457Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"49f43a87-0c8b-4793-ae83-32aa5034dd3b","podName":"counter-85b5h-head-tnx7c","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:39:05.457Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"49f43a87-0c8b-4793-ae83-32aa5034dd3b","podName":"counter-85b5h-head-tnx7c","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:39:05.457Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"49f43a87-0c8b-4793-ae83-32aa5034dd3b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:39:05.457Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"49f43a87-0c8b-4793-ae83-32aa5034dd3b","seconds":300}
{"level":"info","ts":"2026-06-05T09:39:05.459Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"ac0663ad-8245-4351-8615-474baab344dc"}
{"level":"info","ts":"2026-06-05T09:39:05.459Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"ac0663ad-8245-4351-8615-474baab344dc"}
{"level":"info","ts":"2026-06-05T09:39:05.460Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"ac0663ad-8245-4351-8615-474baab344dc","1 head service found":"counter-85b5h-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:05.460Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"ac0663ad-8245-4351-8615-474baab344dc","Found 1 head Pod":"counter-85b5h-head-tnx7c","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:39:05.460Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"ac0663ad-8245-4351-8615-474baab344dc","head Pod":"counter-85b5h-head-tnx7c","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-85b5h-head-tnx7c. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:05.460Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"ac0663ad-8245-4351-8615-474baab344dc","podName":"counter-85b5h-head-tnx7c","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:39:05.460Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"ac0663ad-8245-4351-8615-474baab344dc","podName":"counter-85b5h-head-tnx7c","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:39:05.460Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"ac0663ad-8245-4351-8615-474baab344dc","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:39:05.460Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"ac0663ad-8245-4351-8615-474baab344dc","seconds":300}
{"level":"info","ts":"2026-06-05T09:39:05.463Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"88285706-209a-4ece-bbab-a2b3bb460d2e","name":"counter-85b5h-head"}
{"level":"info","ts":"2026-06-05T09:39:05.487Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"be0d1641-aca5-44cc-9132-5b36e5718f5f"}
{"level":"info","ts":"2026-06-05T09:39:05.487Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"be0d1641-aca5-44cc-9132-5b36e5718f5f"}
{"level":"info","ts":"2026-06-05T09:39:05.487Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"be0d1641-aca5-44cc-9132-5b36e5718f5f","1 head service found":"counter-85b5h-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:05.487Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"be0d1641-aca5-44cc-9132-5b36e5718f5f","Found 1 head Pod":"counter-85b5h-head-tnx7c","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:39:05.487Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"be0d1641-aca5-44cc-9132-5b36e5718f5f","head Pod":"counter-85b5h-head-tnx7c","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-85b5h-head-tnx7c. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:05.488Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"be0d1641-aca5-44cc-9132-5b36e5718f5f","podName":"counter-85b5h-head-tnx7c","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:39:05.488Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"be0d1641-aca5-44cc-9132-5b36e5718f5f","podName":"counter-85b5h-head-tnx7c","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:39:05.488Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"be0d1641-aca5-44cc-9132-5b36e5718f5f","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-05T09:39:05.488Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"be0d1641-aca5-44cc-9132-5b36e5718f5f","name":"counter-85b5h","old status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-85b5h-head-tnx7c","serviceName":"counter-85b5h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-85b5h-head-tnx7c","serviceName":"counter-85b5h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:39:05.488Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"88285706-209a-4ece-bbab-a2b3bb460d2e","name":"counter-85b5h-workers"}
{"level":"info","ts":"2026-06-05T09:39:05.488Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"88285706-209a-4ece-bbab-a2b3bb460d2e","cluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:05.506Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"7856e698-d7d9-4e79-983d-4b975f0117fc","rayCluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:05.506Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"f944f8c4-2b0e-4ea7-8aa0-6767dcaff225","cluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:05.506Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"7856e698-d7d9-4e79-983d-4b975f0117fc","rayCluster":"counter-85b5h","podIPs":[]}
{"level":"info","ts":"2026-06-05T09:39:05.506Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"c582728c-ff6d-4113-ae3c-8ec7da5be326","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:05.506Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"c582728c-ff6d-4113-ae3c-8ec7da5be326","name":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:05.506Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"c582728c-ff6d-4113-ae3c-8ec7da5be326","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:05.506Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"c582728c-ff6d-4113-ae3c-8ec7da5be326"}
{"level":"info","ts":"2026-06-05T09:39:05.506Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"c582728c-ff6d-4113-ae3c-8ec7da5be326","RayCluster":"counter-85b5h","State":""}
{"level":"info","ts":"2026-06-05T09:39:05.521Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"f944f8c4-2b0e-4ea7-8aa0-6767dcaff225","name":"counter-85b5h-head"}
{"level":"info","ts":"2026-06-05T09:39:05.532Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"f944f8c4-2b0e-4ea7-8aa0-6767dcaff225","name":"counter-85b5h-workers"}
{"level":"info","ts":"2026-06-05T09:39:05.532Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"f944f8c4-2b0e-4ea7-8aa0-6767dcaff225","cluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:05.750Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"d69c86e6-7698-4f0f-871c-1ffb02317cc7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:05.750Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"d69c86e6-7698-4f0f-871c-1ffb02317cc7","name":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:05.750Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"d69c86e6-7698-4f0f-871c-1ffb02317cc7","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:05.750Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"d69c86e6-7698-4f0f-871c-1ffb02317cc7"}
{"level":"info","ts":"2026-06-05T09:39:05.750Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"d69c86e6-7698-4f0f-871c-1ffb02317cc7","RayCluster":"counter-85b5h","State":""}
{"level":"info","ts":"2026-06-05T09:39:05.880Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"3705b87a-c9b0-4cb6-b5ff-203bd0339bd0"}
{"level":"info","ts":"2026-06-05T09:39:05.880Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"3705b87a-c9b0-4cb6-b5ff-203bd0339bd0"}
{"level":"info","ts":"2026-06-05T09:39:05.880Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"3705b87a-c9b0-4cb6-b5ff-203bd0339bd0","1 head service found":"counter-85b5h-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:05.880Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"3705b87a-c9b0-4cb6-b5ff-203bd0339bd0","Found 1 head Pod":"counter-85b5h-head-tnx7c","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:39:05.880Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"3705b87a-c9b0-4cb6-b5ff-203bd0339bd0","head Pod":"counter-85b5h-head-tnx7c","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-85b5h-head-tnx7c. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:05.880Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"3705b87a-c9b0-4cb6-b5ff-203bd0339bd0","podName":"counter-85b5h-head-tnx7c","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:39:05.880Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"3705b87a-c9b0-4cb6-b5ff-203bd0339bd0","podName":"counter-85b5h-head-tnx7c","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:39:05.880Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"3705b87a-c9b0-4cb6-b5ff-203bd0339bd0","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:39:05.880Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"3705b87a-c9b0-4cb6-b5ff-203bd0339bd0","seconds":300}
{"level":"info","ts":"2026-06-05T09:39:06.697Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"d157d76f-f33a-418f-a390-32709e2e67f7"}
{"level":"info","ts":"2026-06-05T09:39:06.697Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"d157d76f-f33a-418f-a390-32709e2e67f7"}
{"level":"info","ts":"2026-06-05T09:39:06.697Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"d157d76f-f33a-418f-a390-32709e2e67f7","1 head service found":"counter-85b5h-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:06.698Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"d157d76f-f33a-418f-a390-32709e2e67f7","Found 1 head Pod":"counter-85b5h-head-tnx7c","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:39:06.698Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"d157d76f-f33a-418f-a390-32709e2e67f7","head Pod":"counter-85b5h-head-tnx7c","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-85b5h-head-tnx7c. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:06.698Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"d157d76f-f33a-418f-a390-32709e2e67f7","podName":"counter-85b5h-head-tnx7c","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:39:06.698Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"d157d76f-f33a-418f-a390-32709e2e67f7","podName":"counter-85b5h-head-tnx7c","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:39:06.698Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"d157d76f-f33a-418f-a390-32709e2e67f7","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"counter-85b5h-head-tnx7c","serviceName":"counter-85b5h-head-svc"},"newHead":{"podIP":"10.132.0.15","serviceIP":"10.132.0.15","podName":"counter-85b5h-head-tnx7c","serviceName":"counter-85b5h-head-svc"}}
{"level":"info","ts":"2026-06-05T09:39:06.698Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"d157d76f-f33a-418f-a390-32709e2e67f7","name":"counter-85b5h","old status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-85b5h-head-tnx7c","serviceName":"counter-85b5h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:06Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.15","serviceIP":"10.132.0.15","podName":"counter-85b5h-head-tnx7c","serviceName":"counter-85b5h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:39:06.710Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"b441acbf-bd23-4c90-a117-c6aab512c80e","rayCluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:06.710Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"b441acbf-bd23-4c90-a117-c6aab512c80e","pod":"counter-85b5h-head-tnx7c","ip":"10.132.0.15"}
{"level":"info","ts":"2026-06-05T09:39:06.710Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"2dc4aac2-aa6c-446f-bc21-5a963361dfea","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:06.710Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"2dc4aac2-aa6c-446f-bc21-5a963361dfea","name":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:06.710Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"2dc4aac2-aa6c-446f-bc21-5a963361dfea","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:06.710Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"23d15419-8aab-4323-9cd9-d7f6dfe9b468","cluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:06.710Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"2dc4aac2-aa6c-446f-bc21-5a963361dfea"}
{"level":"info","ts":"2026-06-05T09:39:06.710Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"2dc4aac2-aa6c-446f-bc21-5a963361dfea","RayCluster":"counter-85b5h","State":""}
{"level":"info","ts":"2026-06-05T09:39:06.723Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"23d15419-8aab-4323-9cd9-d7f6dfe9b468","name":"counter-85b5h-head"}
{"level":"info","ts":"2026-06-05T09:39:06.724Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"b441acbf-bd23-4c90-a117-c6aab512c80e","certificate":"ray-head-cert-counter-85b5h","dnsNames":["counter-85b5h-head-svc","localhost","counter-85b5h-head-svc.test-ns-l4cdh.svc","counter-85b5h-head-svc.test-ns-l4cdh.svc.cluster.local"],"ipAddresses":["10.132.0.15","127.0.0.1"]}
{"level":"info","ts":"2026-06-05T09:39:06.736Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"23d15419-8aab-4323-9cd9-d7f6dfe9b468","name":"counter-85b5h-workers"}
{"level":"info","ts":"2026-06-05T09:39:06.736Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"23d15419-8aab-4323-9cd9-d7f6dfe9b468","cluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:06.737Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"b441acbf-bd23-4c90-a117-c6aab512c80e","certificate":"ray-worker-cert-counter-85b5h","dnsNames":["counter-85b5h-worker-svc","localhost","counter-85b5h-worker-svc.test-ns-l4cdh.svc","counter-85b5h-worker-svc.test-ns-l4cdh.svc.cluster.local","*.counter-85b5h-worker-svc.test-ns-l4cdh.svc","*.counter-85b5h-worker-svc.test-ns-l4cdh.svc.cluster.local","*-worker-*.test-ns-l4cdh.svc","*-worker-*.test-ns-l4cdh.svc.cluster.local"],"ipAddresses":["10.132.0.15","127.0.0.1"]}
{"level":"info","ts":"2026-06-05T09:39:06.737Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"b441acbf-bd23-4c90-a117-c6aab512c80e","rayCluster":"counter-85b5h","podIPs":["10.132.0.15"]}
{"level":"info","ts":"2026-06-05T09:39:07.507Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"511d3030-99f2-4960-8b1e-c050ee457e4e"}
{"level":"info","ts":"2026-06-05T09:39:07.507Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"511d3030-99f2-4960-8b1e-c050ee457e4e"}
{"level":"info","ts":"2026-06-05T09:39:07.507Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"511d3030-99f2-4960-8b1e-c050ee457e4e","1 head service found":"counter-85b5h-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:07.507Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"511d3030-99f2-4960-8b1e-c050ee457e4e","Found 1 head Pod":"counter-85b5h-head-tnx7c","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:39:07.507Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"511d3030-99f2-4960-8b1e-c050ee457e4e","head Pod":"counter-85b5h-head-tnx7c","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-85b5h-head-tnx7c. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:07.507Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"511d3030-99f2-4960-8b1e-c050ee457e4e","podName":"counter-85b5h-head-tnx7c","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:39:07.507Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"511d3030-99f2-4960-8b1e-c050ee457e4e","podName":"counter-85b5h-head-tnx7c","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:39:07.507Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"511d3030-99f2-4960-8b1e-c050ee457e4e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:39:07.507Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"511d3030-99f2-4960-8b1e-c050ee457e4e","seconds":300}
{"level":"info","ts":"2026-06-05T09:39:08.751Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"5989c9ef-85c2-4aae-98a0-f7a289a60a2b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:08.751Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"5989c9ef-85c2-4aae-98a0-f7a289a60a2b","name":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:08.751Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"5989c9ef-85c2-4aae-98a0-f7a289a60a2b","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:08.751Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"5989c9ef-85c2-4aae-98a0-f7a289a60a2b"}
{"level":"info","ts":"2026-06-05T09:39:08.751Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"5989c9ef-85c2-4aae-98a0-f7a289a60a2b","RayCluster":"counter-85b5h","State":""}
{"level":"info","ts":"2026-06-05T09:39:11.719Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"668ac855-234b-4cf1-bc48-8bd2581eff5d"}
{"level":"info","ts":"2026-06-05T09:39:11.719Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"668ac855-234b-4cf1-bc48-8bd2581eff5d"}
{"level":"info","ts":"2026-06-05T09:39:11.719Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"668ac855-234b-4cf1-bc48-8bd2581eff5d","1 head service found":"counter-85b5h-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:11.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"668ac855-234b-4cf1-bc48-8bd2581eff5d","Found 1 head Pod":"counter-85b5h-head-tnx7c","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:39:11.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"668ac855-234b-4cf1-bc48-8bd2581eff5d","head Pod":"counter-85b5h-head-tnx7c","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-85b5h-head-tnx7c. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:11.720Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"668ac855-234b-4cf1-bc48-8bd2581eff5d","podName":"counter-85b5h-head-tnx7c","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:39:11.720Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"668ac855-234b-4cf1-bc48-8bd2581eff5d","podName":"counter-85b5h-head-tnx7c","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:39:11.720Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"668ac855-234b-4cf1-bc48-8bd2581eff5d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:39:11.720Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"668ac855-234b-4cf1-bc48-8bd2581eff5d","seconds":300}
{"level":"info","ts":"2026-06-05T09:39:11.752Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"07f7f8db-2847-4652-b43d-07b0dc248ad2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:11.752Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"07f7f8db-2847-4652-b43d-07b0dc248ad2","name":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:11.752Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"07f7f8db-2847-4652-b43d-07b0dc248ad2","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:11.752Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"07f7f8db-2847-4652-b43d-07b0dc248ad2"}
{"level":"info","ts":"2026-06-05T09:39:11.752Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"07f7f8db-2847-4652-b43d-07b0dc248ad2","RayCluster":"counter-85b5h","State":""}
{"level":"info","ts":"2026-06-05T09:39:12.724Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"b63e95a5-8b49-4eef-86e6-9f16699da9df"}
{"level":"info","ts":"2026-06-05T09:39:12.724Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"b63e95a5-8b49-4eef-86e6-9f16699da9df"}
{"level":"info","ts":"2026-06-05T09:39:12.724Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"b63e95a5-8b49-4eef-86e6-9f16699da9df","1 head service found":"counter-85b5h-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:12.724Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"b63e95a5-8b49-4eef-86e6-9f16699da9df","Found 1 head Pod":"counter-85b5h-head-tnx7c","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:39:12.724Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"b63e95a5-8b49-4eef-86e6-9f16699da9df","head Pod":"counter-85b5h-head-tnx7c","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-85b5h-head-tnx7c. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:12.724Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"b63e95a5-8b49-4eef-86e6-9f16699da9df","podName":"counter-85b5h-head-tnx7c","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:39:05Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:39:12.724Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"b63e95a5-8b49-4eef-86e6-9f16699da9df","podName":"counter-85b5h-head-tnx7c","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:39:05Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:39:12.724Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"b63e95a5-8b49-4eef-86e6-9f16699da9df","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-05T09:39:12.724Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"b63e95a5-8b49-4eef-86e6-9f16699da9df","name":"counter-85b5h","old status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:06Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.15","serviceIP":"10.132.0.15","podName":"counter-85b5h-head-tnx7c","serviceName":"counter-85b5h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.15","serviceIP":"10.132.0.15","podName":"counter-85b5h-head-tnx7c","serviceName":"counter-85b5h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:39:12.738Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"94231b18-3200-4a56-986f-929e51d17f6b","rayCluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:12.738Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"40358add-6e77-4339-bc70-06273879912c","cluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:12.738Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"94231b18-3200-4a56-986f-929e51d17f6b","pod":"counter-85b5h-head-tnx7c","ip":"10.132.0.15"}
{"level":"info","ts":"2026-06-05T09:39:12.738Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"a096ca83-08ce-4def-8ef7-c57e2ade421c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:12.738Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"a096ca83-08ce-4def-8ef7-c57e2ade421c","name":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:12.738Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"94231b18-3200-4a56-986f-929e51d17f6b","rayCluster":"counter-85b5h","podIPs":["10.132.0.15"]}
{"level":"info","ts":"2026-06-05T09:39:12.738Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"a096ca83-08ce-4def-8ef7-c57e2ade421c","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:12.738Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"a096ca83-08ce-4def-8ef7-c57e2ade421c"}
{"level":"info","ts":"2026-06-05T09:39:12.738Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"a096ca83-08ce-4def-8ef7-c57e2ade421c","RayCluster":"counter-85b5h","State":""}
{"level":"info","ts":"2026-06-05T09:39:12.751Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"40358add-6e77-4339-bc70-06273879912c","name":"counter-85b5h-head"}
{"level":"info","ts":"2026-06-05T09:39:12.761Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"40358add-6e77-4339-bc70-06273879912c","name":"counter-85b5h-workers"}
{"level":"info","ts":"2026-06-05T09:39:12.761Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"40358add-6e77-4339-bc70-06273879912c","cluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:14.738Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"0b133982-0a99-4086-a991-86961248014a"}
{"level":"info","ts":"2026-06-05T09:39:14.738Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"0b133982-0a99-4086-a991-86961248014a"}
{"level":"info","ts":"2026-06-05T09:39:14.738Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"0b133982-0a99-4086-a991-86961248014a","1 head service found":"counter-85b5h-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:14.739Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"0b133982-0a99-4086-a991-86961248014a","Found 1 head Pod":"counter-85b5h-head-tnx7c","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:39:14.739Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"0b133982-0a99-4086-a991-86961248014a","head Pod":"counter-85b5h-head-tnx7c","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-85b5h-head-tnx7c. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:14.739Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"0b133982-0a99-4086-a991-86961248014a","podName":"counter-85b5h-head-tnx7c","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:39:05Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:39:14.739Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"0b133982-0a99-4086-a991-86961248014a","podName":"counter-85b5h-head-tnx7c","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:39:05Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:39:14.739Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"0b133982-0a99-4086-a991-86961248014a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:39:14.739Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"0b133982-0a99-4086-a991-86961248014a","seconds":300}
{"level":"info","ts":"2026-06-05T09:39:14.753Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"11d117e2-fbf7-49c8-8a68-ee2c57fad691","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:14.753Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"11d117e2-fbf7-49c8-8a68-ee2c57fad691","name":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:14.753Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"11d117e2-fbf7-49c8-8a68-ee2c57fad691","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:14.753Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"11d117e2-fbf7-49c8-8a68-ee2c57fad691"}
{"level":"info","ts":"2026-06-05T09:39:14.753Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"11d117e2-fbf7-49c8-8a68-ee2c57fad691","RayCluster":"counter-85b5h","State":""}
{"level":"info","ts":"2026-06-05T09:39:17.753Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"04c26461-43fa-4983-b379-8adc583260f5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:17.753Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"04c26461-43fa-4983-b379-8adc583260f5","name":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:17.753Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"04c26461-43fa-4983-b379-8adc583260f5","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:17.753Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"04c26461-43fa-4983-b379-8adc583260f5"}
{"level":"info","ts":"2026-06-05T09:39:17.753Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"04c26461-43fa-4983-b379-8adc583260f5","RayCluster":"counter-85b5h","State":""}
{"level":"info","ts":"2026-06-05T09:39:20.754Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"6babd65b-91a1-4e8c-b6aa-a304258666c1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:20.754Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"6babd65b-91a1-4e8c-b6aa-a304258666c1","name":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:20.754Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"6babd65b-91a1-4e8c-b6aa-a304258666c1","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:20.754Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"6babd65b-91a1-4e8c-b6aa-a304258666c1"}
{"level":"info","ts":"2026-06-05T09:39:20.754Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"6babd65b-91a1-4e8c-b6aa-a304258666c1","RayCluster":"counter-85b5h","State":""}
{"level":"info","ts":"2026-06-05T09:39:23.752Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"e84c9208-97b3-4a70-8ea9-dba1846f7275"}
{"level":"info","ts":"2026-06-05T09:39:23.752Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"e84c9208-97b3-4a70-8ea9-dba1846f7275"}
{"level":"info","ts":"2026-06-05T09:39:23.752Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"e84c9208-97b3-4a70-8ea9-dba1846f7275","1 head service found":"counter-85b5h-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:23.753Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"e84c9208-97b3-4a70-8ea9-dba1846f7275","Found 1 head Pod":"counter-85b5h-head-tnx7c","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:39:23.753Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"e84c9208-97b3-4a70-8ea9-dba1846f7275","head Pod":"counter-85b5h-head-tnx7c","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-85b5h-head-tnx7c. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:23.753Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"e84c9208-97b3-4a70-8ea9-dba1846f7275","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-06-05T09:39:23.753Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"e84c9208-97b3-4a70-8ea9-dba1846f7275","name":"counter-85b5h","old status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.15","serviceIP":"10.132.0.15","podName":"counter-85b5h-head-tnx7c","serviceName":"counter-85b5h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:23Z","stateTransitionTimes":{"ready":"2026-06-05T09:39:23Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.15","serviceIP":"10.132.0.15","podName":"counter-85b5h-head-tnx7c","serviceName":"counter-85b5h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:39:23Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:39:23Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:39:23.754Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"9d7433cd-3a2e-42bb-8688-48b1c6fcb9ef","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:23.754Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"9d7433cd-3a2e-42bb-8688-48b1c6fcb9ef","name":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:23.754Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"9d7433cd-3a2e-42bb-8688-48b1c6fcb9ef","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:23.754Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"9d7433cd-3a2e-42bb-8688-48b1c6fcb9ef"}
{"level":"info","ts":"2026-06-05T09:39:23.754Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"9d7433cd-3a2e-42bb-8688-48b1c6fcb9ef","RayCluster":"counter-85b5h","State":""}
{"level":"info","ts":"2026-06-05T09:39:23.765Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"21bffe7f-8c4c-4589-a9b5-3452c3aceb5e","rayCluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:23.765Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"67a4a696-ff05-460d-87b6-a6aceb9571e4","cluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:23.765Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"21bffe7f-8c4c-4589-a9b5-3452c3aceb5e","pod":"counter-85b5h-head-tnx7c","ip":"10.132.0.15"}
{"level":"info","ts":"2026-06-05T09:39:23.765Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"21bffe7f-8c4c-4589-a9b5-3452c3aceb5e","rayCluster":"counter-85b5h","podIPs":["10.132.0.15"]}
{"level":"info","ts":"2026-06-05T09:39:23.765Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"230b66b5-5f6c-40cc-bd53-e6d71dd1bd6b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:23.765Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"230b66b5-5f6c-40cc-bd53-e6d71dd1bd6b","name":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:23.765Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"230b66b5-5f6c-40cc-bd53-e6d71dd1bd6b","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:23.766Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"230b66b5-5f6c-40cc-bd53-e6d71dd1bd6b"}
{"level":"info","ts":"2026-06-05T09:39:23.766Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"230b66b5-5f6c-40cc-bd53-e6d71dd1bd6b","head service name":"counter-85b5h-head-svc","namespace":"test-ns-l4cdh"}
{"level":"info","ts":"2026-06-05T09:39:23.766Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"230b66b5-5f6c-40cc-bd53-e6d71dd1bd6b","head service URL":"counter-85b5h-head-svc.test-ns-l4cdh.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-05T09:39:23.766Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"230b66b5-5f6c-40cc-bd53-e6d71dd1bd6b","SubmissionMode":"HTTPMode","RayCluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:23.766Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"230b66b5-5f6c-40cc-bd53-e6d71dd1bd6b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5mm55","rayClusterName":"counter-85b5h","jobDeploymentStatus":"Initializing","startTime":"2026-06-05T09:39:02Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5mm55","rayClusterName":"counter-85b5h","dashboardURL":"counter-85b5h-head-svc.test-ns-l4cdh.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-05T09:39:02Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:39:23.766Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"230b66b5-5f6c-40cc-bd53-e6d71dd1bd6b","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-05T09:39:23.776Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"15c2f4ec-6042-4873-b1f1-75455873ff49","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:23.776Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"15c2f4ec-6042-4873-b1f1-75455873ff49","name":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:23.776Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"15c2f4ec-6042-4873-b1f1-75455873ff49","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:23.777Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"15c2f4ec-6042-4873-b1f1-75455873ff49"}
{"level":"info","ts":"2026-06-05T09:39:23.906Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"67a4a696-ff05-460d-87b6-a6aceb9571e4","name":"counter-85b5h-head"}
{"level":"info","ts":"2026-06-05T09:39:23.916Z","logger":"controllers.RayJob","msg":"The Ray job was not found. Submit a Ray job via an HTTP request.","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"15c2f4ec-6042-4873-b1f1-75455873ff49","JobId":"counter-5mm55"}
{"level":"info","ts":"2026-06-05T09:39:23.916Z","logger":"controllers.RayJob","msg":"Submit a ray job","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"15c2f4ec-6042-4873-b1f1-75455873ff49","rayJob":"counter","jobInfo":"{\"runtime_env\":{\"env_vars\":{\"counter_name\":\"test_counter\"}},\"entrypoint_resources\":{\"R1\":2},\"entrypoint\":\"python /home/ray/jobs/counter.py\",\"submission_id\":\"counter-5mm55\",\"entrypoint_num_cpus\":2,\"entrypoint_num_gpus\":2}"}
{"level":"info","ts":"2026-06-05T09:39:23.919Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"67a4a696-ff05-460d-87b6-a6aceb9571e4","name":"counter-85b5h-workers"}
{"level":"info","ts":"2026-06-05T09:39:23.919Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"67a4a696-ff05-460d-87b6-a6aceb9571e4","cluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:25.766Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"06a860b2-9816-4a54-b107-0d4df1a6011a"}
{"level":"info","ts":"2026-06-05T09:39:25.766Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"06a860b2-9816-4a54-b107-0d4df1a6011a"}
{"level":"info","ts":"2026-06-05T09:39:25.766Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"06a860b2-9816-4a54-b107-0d4df1a6011a","1 head service found":"counter-85b5h-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:25.766Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"06a860b2-9816-4a54-b107-0d4df1a6011a","Found 1 head Pod":"counter-85b5h-head-tnx7c","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:39:25.766Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"06a860b2-9816-4a54-b107-0d4df1a6011a","head Pod":"counter-85b5h-head-tnx7c","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-85b5h-head-tnx7c. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:25.766Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"06a860b2-9816-4a54-b107-0d4df1a6011a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:39:25.766Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"06a860b2-9816-4a54-b107-0d4df1a6011a","seconds":300}
{"level":"info","ts":"2026-06-05T09:39:26.755Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"08fea77e-c218-4c9e-8814-881f82c30d2f","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:26.755Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"08fea77e-c218-4c9e-8814-881f82c30d2f","name":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:26.755Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"08fea77e-c218-4c9e-8814-881f82c30d2f","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:26.755Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"08fea77e-c218-4c9e-8814-881f82c30d2f"}
{"level":"info","ts":"2026-06-05T09:39:26.759Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"08fea77e-c218-4c9e-8814-881f82c30d2f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5mm55","rayClusterName":"counter-85b5h","dashboardURL":"counter-85b5h-head-svc.test-ns-l4cdh.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-05T09:39:02Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:39:24Z"},"jobId":"counter-5mm55","rayClusterName":"counter-85b5h","dashboardURL":"counter-85b5h-head-svc.test-ns-l4cdh.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-05T09:39:02Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:23Z","stateTransitionTimes":{"ready":"2026-06-05T09:39:23Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.15","serviceIP":"10.132.0.15","podName":"counter-85b5h-head-tnx7c","serviceName":"counter-85b5h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:39:23Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:39:23Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:39:26.759Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"08fea77e-c218-4c9e-8814-881f82c30d2f","old JobStatus":"","new JobStatus":"RUNNING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-05T09:39:26.773Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"880ba0d1-7627-4fa6-aed1-715e9774932a","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:26.773Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"880ba0d1-7627-4fa6-aed1-715e9774932a","name":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:26.773Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"880ba0d1-7627-4fa6-aed1-715e9774932a","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:26.773Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"880ba0d1-7627-4fa6-aed1-715e9774932a"}
{"level":"info","ts":"2026-06-05T09:39:26.776Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"880ba0d1-7627-4fa6-aed1-715e9774932a","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:39:24Z"},"jobId":"counter-5mm55","rayClusterName":"counter-85b5h","dashboardURL":"counter-85b5h-head-svc.test-ns-l4cdh.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-05T09:39:02Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:23Z","stateTransitionTimes":{"ready":"2026-06-05T09:39:23Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.15","serviceIP":"10.132.0.15","podName":"counter-85b5h-head-tnx7c","serviceName":"counter-85b5h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:39:23Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:39:23Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:39:24Z"},"jobId":"counter-5mm55","rayClusterName":"counter-85b5h","dashboardURL":"counter-85b5h-head-svc.test-ns-l4cdh.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-05T09:39:02Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:23Z","stateTransitionTimes":{"ready":"2026-06-05T09:39:23Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.15","serviceIP":"10.132.0.15","podName":"counter-85b5h-head-tnx7c","serviceName":"counter-85b5h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:39:23Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:39:23Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:39:29.773Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"c98868f6-5618-4c0d-8896-8a94147e3f81","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:29.773Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"c98868f6-5618-4c0d-8896-8a94147e3f81","name":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:29.773Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"c98868f6-5618-4c0d-8896-8a94147e3f81","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:29.774Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"c98868f6-5618-4c0d-8896-8a94147e3f81"}
{"level":"info","ts":"2026-06-05T09:39:29.777Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"c98868f6-5618-4c0d-8896-8a94147e3f81","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:39:24Z"},"jobId":"counter-5mm55","rayClusterName":"counter-85b5h","dashboardURL":"counter-85b5h-head-svc.test-ns-l4cdh.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-05T09:39:02Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:23Z","stateTransitionTimes":{"ready":"2026-06-05T09:39:23Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.15","serviceIP":"10.132.0.15","podName":"counter-85b5h-head-tnx7c","serviceName":"counter-85b5h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:39:23Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:39:23Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:39:24Z","endTime":"2026-06-05T09:39:29Z"},"jobId":"counter-5mm55","rayClusterName":"counter-85b5h","dashboardURL":"counter-85b5h-head-svc.test-ns-l4cdh.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Complete","message":"Job finished successfully.","startTime":"2026-06-05T09:39:02Z","succeeded":1,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:23Z","stateTransitionTimes":{"ready":"2026-06-05T09:39:23Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.15","serviceIP":"10.132.0.15","podName":"counter-85b5h-head-tnx7c","serviceName":"counter-85b5h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:39:23Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:39:23Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:02Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:39:29.777Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"c98868f6-5618-4c0d-8896-8a94147e3f81","old JobStatus":"RUNNING","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Complete"}
{"level":"info","ts":"2026-06-05T09:39:29.793Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"767ee688-9648-4c8e-b7ea-33960470901a","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:29.793Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"767ee688-9648-4c8e-b7ea-33960470901a","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-05 09:39:29 +0000 UTC","Now":"2026-06-05T09:39:29.793Z","ShutdownTime":"2026-06-05T09:39:29.000Z"}
{"level":"info","ts":"2026-06-05T09:39:29.793Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"767ee688-9648-4c8e-b7ea-33960470901a","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-05T09:39:29.802Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob is deleted","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"767ee688-9648-4c8e-b7ea-33960470901a","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:29.802Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"767ee688-9648-4c8e-b7ea-33960470901a","isClusterDeleted":false}
{"level":"info","ts":"2026-06-05T09:39:29.802Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"767ee688-9648-4c8e-b7ea-33960470901a","RayCluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:29.803Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"b2bb9620-de9c-465e-925f-5f2e8bff6d25"}
{"level":"info","ts":"2026-06-05T09:39:29.803Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"f65332f7-cd92-4535-81b6-734c00f73440","namespacedName":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:29.803Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"4683f606-03d8-4cfb-8426-f83d878d0c7c","rayCluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:29.803Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"f65332f7-cd92-4535-81b6-734c00f73440","cluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:29.803Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"f65332f7-cd92-4535-81b6-734c00f73440","cluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:29.803Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"reconcileID":"46101bb6-4be9-4e87-a946-c45aa46c9739"}
{"level":"info","ts":"2026-06-05T09:39:29.803Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"4683f606-03d8-4cfb-8426-f83d878d0c7c","pod":"counter-85b5h-head-tnx7c","ip":"10.132.0.15"}
{"level":"info","ts":"2026-06-05T09:39:29.803Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"f65332f7-cd92-4535-81b6-734c00f73440","namespace":"test-ns-l4cdh","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-05T09:39:29.803Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"4683f606-03d8-4cfb-8426-f83d878d0c7c","rayCluster":"counter-85b5h","podIPs":["10.132.0.15"]}
{"level":"info","ts":"2026-06-05T09:39:29.803Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"9a970f40-f376-4552-b261-2a5ab3dcffd9","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:29.803Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"9a970f40-f376-4552-b261-2a5ab3dcffd9","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-05 09:39:29 +0000 UTC","Now":"2026-06-05T09:39:29.803Z","ShutdownTime":"2026-06-05T09:39:29.000Z"}
{"level":"info","ts":"2026-06-05T09:39:29.803Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"9a970f40-f376-4552-b261-2a5ab3dcffd9","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-05T09:39:29.803Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"9a970f40-f376-4552-b261-2a5ab3dcffd9","RayCluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:29.803Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"9a970f40-f376-4552-b261-2a5ab3dcffd9","isClusterDeleted":false}
{"level":"info","ts":"2026-06-05T09:39:29.803Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"9a970f40-f376-4552-b261-2a5ab3dcffd9","RayCluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:29.809Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"f65332f7-cd92-4535-81b6-734c00f73440","name":"kuberay-gateway-access","namespace":"test-ns-l4cdh"}
{"level":"info","ts":"2026-06-05T09:39:29.817Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"f65332f7-cd92-4535-81b6-734c00f73440","name":"test-ns-l4cdh-counter-85b5h","namespace":"opendatahub"}
{"level":"info","ts":"2026-06-05T09:39:29.823Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"f65332f7-cd92-4535-81b6-734c00f73440","configMap":"kube-rbac-proxy-config-counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:29.828Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"f65332f7-cd92-4535-81b6-734c00f73440","serviceAccount":"counter-85b5h-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:39:29.834Z","logger":"raycluster-resource","msg":"default","name":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:29.834Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-85b5h","namespace":"test-ns-l4cdh"}
{"level":"info","ts":"2026-06-05T09:39:29.844Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"f65332f7-cd92-4535-81b6-734c00f73440","cluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:29.844Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"028f5dd3-90ae-4260-bb54-108098018843","namespacedName":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:29.844Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"028f5dd3-90ae-4260-bb54-108098018843","cluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:29.844Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"028f5dd3-90ae-4260-bb54-108098018843","cluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:29.844Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"028f5dd3-90ae-4260-bb54-108098018843","namespace":"test-ns-l4cdh","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-05T09:39:29.844Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"028f5dd3-90ae-4260-bb54-108098018843","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-06-05T09:39:29.844Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"aab8ceff-c2af-4bd2-9020-d8db372a2032"}
{"level":"info","ts":"2026-06-05T09:39:29.844Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"51ff4779-59cc-4088-b5cf-1857ff6d522f","namespace":"test-ns-l4cdh","clusterName":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:29.844Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"13d1036c-e3b8-4c26-8164-6f1ef6b72be4","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:29.844Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"13d1036c-e3b8-4c26-8164-6f1ef6b72be4","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-05 09:39:29 +0000 UTC","Now":"2026-06-05T09:39:29.844Z","ShutdownTime":"2026-06-05T09:39:29.000Z"}
{"level":"info","ts":"2026-06-05T09:39:29.844Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"13d1036c-e3b8-4c26-8164-6f1ef6b72be4","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-05T09:39:29.844Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"13d1036c-e3b8-4c26-8164-6f1ef6b72be4","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:29.844Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"13d1036c-e3b8-4c26-8164-6f1ef6b72be4","isClusterDeleted":true}
{"level":"info","ts":"2026-06-05T09:39:29.844Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"13d1036c-e3b8-4c26-8164-6f1ef6b72be4","RayCluster":"counter-85b5h"}
{"level":"error","ts":"2026-06-05T09:39:29.849Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"028f5dd3-90ae-4260-bb54-108098018843","error":"rayclusters.ray.io \"counter-85b5h\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:214\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:155\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:29.849Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"028f5dd3-90ae-4260-bb54-108098018843"}
{"level":"error","ts":"2026-06-05T09:39:29.849Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"028f5dd3-90ae-4260-bb54-108098018843","error":"rayclusters.ray.io \"counter-85b5h\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:29.849Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"8f6e3a4f-e591-47de-bfc0-2864a5a74042","namespacedName":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:29.849Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"8f6e3a4f-e591-47de-bfc0-2864a5a74042"}
{"level":"info","ts":"2026-06-05T09:39:29.849Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"8f6e3a4f-e591-47de-bfc0-2864a5a74042","namespace":"test-ns-l4cdh","authClusters":0}
{"level":"info","ts":"2026-06-05T09:39:29.849Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"8f6e3a4f-e591-47de-bfc0-2864a5a74042","namespace":"test-ns-l4cdh"}
{"level":"info","ts":"2026-06-05T09:39:29.853Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"164c7a19-735e-40ec-8942-62a9888315a5"}
{"level":"info","ts":"2026-06-05T09:39:29.854Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"0fd5d15d-b90a-4fec-98e9-c52ad2f865f8"}
{"level":"info","ts":"2026-06-05T09:39:29.855Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"0242af74-2584-4e76-8f1d-c1860f83f0a7","namespacedName":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:29.855Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"0242af74-2584-4e76-8f1d-c1860f83f0a7"}
{"level":"info","ts":"2026-06-05T09:39:29.855Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"0242af74-2584-4e76-8f1d-c1860f83f0a7","namespace":"test-ns-l4cdh","authClusters":0}
{"level":"info","ts":"2026-06-05T09:39:29.855Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"0242af74-2584-4e76-8f1d-c1860f83f0a7","namespace":"test-ns-l4cdh"}
{"level":"info","ts":"2026-06-05T09:39:29.906Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"7ed4bac4-9822-4e04-9763-b2090d25eff2","namespacedName":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:29.906Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"7ed4bac4-9822-4e04-9763-b2090d25eff2"}
{"level":"info","ts":"2026-06-05T09:39:29.906Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"7ed4bac4-9822-4e04-9763-b2090d25eff2","namespace":"test-ns-l4cdh","authClusters":0}
{"level":"info","ts":"2026-06-05T09:39:29.906Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"7ed4bac4-9822-4e04-9763-b2090d25eff2","namespace":"test-ns-l4cdh"}
{"level":"error","ts":"2026-06-05T09:39:29.910Z","msg":"Failed to delete CA certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"51ff4779-59cc-4088-b5cf-1857ff6d522f","certificate":"ray-ca-certificate-counter-85b5h","error":"certificates.cert-manager.io \"ray-ca-certificate-counter-85b5h\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:275\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:29.911Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"e533dd9c-bca5-4010-ad13-23c6804232a9","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:29.911Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"e533dd9c-bca5-4010-ad13-23c6804232a9","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-05T09:39:29.921Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"e533dd9c-bca5-4010-ad13-23c6804232a9"}
{"level":"info","ts":"2026-06-05T09:39:29.921Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"e533dd9c-bca5-4010-ad13-23c6804232a9","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-05T09:39:29.921Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"e533dd9c-bca5-4010-ad13-23c6804232a9","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-t7t9k","rayClusterName":"fail-d7vcr","jobDeploymentStatus":"Initializing","startTime":"2026-06-05T09:39:29Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:39:29.921Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"e533dd9c-bca5-4010-ad13-23c6804232a9","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-05T09:39:29.928Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"51ff4779-59cc-4088-b5cf-1857ff6d522f","errorCount":1}
{"level":"info","ts":"2026-06-05T09:39:29.928Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"51ff4779-59cc-4088-b5cf-1857ff6d522f"}
{"level":"error","ts":"2026-06-05T09:39:29.928Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"51ff4779-59cc-4088-b5cf-1857ff6d522f","error":"certificates.cert-manager.io \"ray-ca-certificate-counter-85b5h\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:29.932Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"b58872cd-5f62-47a3-ad2f-aa661952c52a","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:29.932Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"b58872cd-5f62-47a3-ad2f-aa661952c52a"}
{"level":"info","ts":"2026-06-05T09:39:29.932Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"b58872cd-5f62-47a3-ad2f-aa661952c52a","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-05T09:39:29.932Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"b58872cd-5f62-47a3-ad2f-aa661952c52a","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-kzdbn","rayClusterName":"fail-qwzrw","jobDeploymentStatus":"Initializing","startTime":"2026-06-05T09:39:29Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:39:29.932Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"b58872cd-5f62-47a3-ad2f-aa661952c52a","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-05T09:39:29.934Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"826856fe-7851-4b38-a575-fdf47b4c5150","namespace":"test-ns-l4cdh","clusterName":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:29.934Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"counter-85b5h","reconcileID":"826856fe-7851-4b38-a575-fdf47b4c5150","namespace":"test-ns-l4cdh","clusterName":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:29.938Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"b58872cd-5f62-47a3-ad2f-aa661952c52a","error":"Operation cannot be fulfilled on rayjobs.ray.io \"fail\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-06-05T09:39:29.938Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"b58872cd-5f62-47a3-ad2f-aa661952c52a"}
{"level":"error","ts":"2026-06-05T09:39:29.938Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"b58872cd-5f62-47a3-ad2f-aa661952c52a","error":"Operation cannot be fulfilled on rayjobs.ray.io \"fail\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:29.938Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"8fedc301-1c9d-4a8f-a07a-5a2925d2838c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:29.938Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"8fedc301-1c9d-4a8f-a07a-5a2925d2838c","name":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:29.938Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"8fedc301-1c9d-4a8f-a07a-5a2925d2838c","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:29.938Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"8fedc301-1c9d-4a8f-a07a-5a2925d2838c","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:29.943Z","logger":"raycluster-resource","msg":"default","name":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:29.943Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-d7vcr","namespace":"test-ns-l4cdh"}
{"level":"info","ts":"2026-06-05T09:39:29.951Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"8fedc301-1c9d-4a8f-a07a-5a2925d2838c","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:29.951Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"915ba1dc-689d-498e-af4d-1a5bfceaf12e","rayCluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:29.951Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"915ba1dc-689d-498e-af4d-1a5bfceaf12e","rayCluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:29.951Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"c30af9b5-2602-49b8-93ba-33dd91260ab1","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:29.951Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"f12b713e-8f05-46c8-afa6-9cf497909d28","namespacedName":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:29.951Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"654a1b2a-fca7-4b71-a341-ca45970f6c7e"}
{"level":"info","ts":"2026-06-05T09:39:29.951Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"654a1b2a-fca7-4b71-a341-ca45970f6c7e"}
{"level":"info","ts":"2026-06-05T09:39:29.951Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"f12b713e-8f05-46c8-afa6-9cf497909d28","mode":"IntegratedOAuth","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:29.951Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"f12b713e-8f05-46c8-afa6-9cf497909d28","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:29.951Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"f12b713e-8f05-46c8-afa6-9cf497909d28","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:29.951Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"8fedc301-1c9d-4a8f-a07a-5a2925d2838c"}
{"level":"info","ts":"2026-06-05T09:39:29.951Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"8fedc301-1c9d-4a8f-a07a-5a2925d2838c","RayCluster":"fail-d7vcr","State":""}
{"level":"info","ts":"2026-06-05T09:39:29.951Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"55fa276b-2f44-432e-ad4a-4cf7d858f2f1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:30.005Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"55fa276b-2f44-432e-ad4a-4cf7d858f2f1","name":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:30.005Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"55fa276b-2f44-432e-ad4a-4cf7d858f2f1","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:30.006Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"55fa276b-2f44-432e-ad4a-4cf7d858f2f1"}
{"level":"info","ts":"2026-06-05T09:39:30.006Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"915ba1dc-689d-498e-af4d-1a5bfceaf12e","rayCluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:30.006Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"55fa276b-2f44-432e-ad4a-4cf7d858f2f1","RayCluster":"fail-d7vcr","State":""}
{"level":"info","ts":"2026-06-05T09:39:30.006Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"915ba1dc-689d-498e-af4d-1a5bfceaf12e","rayCluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:30.006Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"654a1b2a-fca7-4b71-a341-ca45970f6c7e","name":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:30.006Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"654a1b2a-fca7-4b71-a341-ca45970f6c7e","error":"head certificate secret ray-head-secret-fail-d7vcr not found"}
{"level":"error","ts":"2026-06-05T09:39:30.006Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"654a1b2a-fca7-4b71-a341-ca45970f6c7e","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-d7vcr not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:30.006Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"654a1b2a-fca7-4b71-a341-ca45970f6c7e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-d7vcr not found"}
{"level":"info","ts":"2026-06-05T09:39:30.006Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"654a1b2a-fca7-4b71-a341-ca45970f6c7e","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"fail-d7vcr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:30.006Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"654a1b2a-fca7-4b71-a341-ca45970f6c7e","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"fail-d7vcr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:30.006Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"654a1b2a-fca7-4b71-a341-ca45970f6c7e","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"fail-d7vcr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:30.006Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"654a1b2a-fca7-4b71-a341-ca45970f6c7e","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-05T09:39:30.006Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"654a1b2a-fca7-4b71-a341-ca45970f6c7e","name":"fail-d7vcr","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-d7vcr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:39:30.012Z","logger":"raycluster-resource","msg":"default","name":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:30.012Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-d7vcr","namespace":"test-ns-l4cdh"}
{"level":"info","ts":"2026-06-05T09:39:30.014Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"c30af9b5-2602-49b8-93ba-33dd91260ab1","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:30.014Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"e48cafc6-c92f-4363-a36e-9b5e0ca96a08","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:30.018Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"915ba1dc-689d-498e-af4d-1a5bfceaf12e"}
{"level":"info","ts":"2026-06-05T09:39:30.018Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"915ba1dc-689d-498e-af4d-1a5bfceaf12e","rayCluster":"fail-d7vcr","secret":"ca-secret"}
{"level":"info","ts":"2026-06-05T09:39:30.018Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"915ba1dc-689d-498e-af4d-1a5bfceaf12e","rayCluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:30.030Z","logger":"raycluster-resource","msg":"default","name":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:30.030Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-d7vcr","namespace":"test-ns-l4cdh"}
{"level":"info","ts":"2026-06-05T09:39:30.105Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"654a1b2a-fca7-4b71-a341-ca45970f6c7e"}
{"level":"error","ts":"2026-06-05T09:39:30.105Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"654a1b2a-fca7-4b71-a341-ca45970f6c7e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-d7vcr not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:30.106Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"915ba1dc-689d-498e-af4d-1a5bfceaf12e","rayCluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:30.106Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"e48cafc6-c92f-4363-a36e-9b5e0ca96a08","name":"fail-d7vcr-head"}
{"level":"error","ts":"2026-06-05T09:39:30.106Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"f12b713e-8f05-46c8-afa6-9cf497909d28","error":"failed to add finalizer: Operation cannot be fulfilled on rayclusters.ray.io \"fail-d7vcr\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:30.106Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"f12b713e-8f05-46c8-afa6-9cf497909d28"}
{"level":"error","ts":"2026-06-05T09:39:30.106Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"f12b713e-8f05-46c8-afa6-9cf497909d28","error":"failed to add finalizer: Operation cannot be fulfilled on rayclusters.ray.io \"fail-d7vcr\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:30.106Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"dab3a8d9-67e8-408e-aec3-e382eef73c34","namespacedName":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:30.106Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"dab3a8d9-67e8-408e-aec3-e382eef73c34","mode":"IntegratedOAuth","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:30.106Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"dab3a8d9-67e8-408e-aec3-e382eef73c34","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:30.106Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"dab3a8d9-67e8-408e-aec3-e382eef73c34","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:30.106Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"4508c7c3-bfdc-43e6-a1b1-8216249f5e1e"}
{"level":"info","ts":"2026-06-05T09:39:30.106Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"4508c7c3-bfdc-43e6-a1b1-8216249f5e1e"}
{"level":"info","ts":"2026-06-05T09:39:30.106Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"4508c7c3-bfdc-43e6-a1b1-8216249f5e1e","1 head service found":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:30.106Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"4508c7c3-bfdc-43e6-a1b1-8216249f5e1e","error":"head certificate secret ray-head-secret-fail-d7vcr not found"}
{"level":"error","ts":"2026-06-05T09:39:30.106Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"4508c7c3-bfdc-43e6-a1b1-8216249f5e1e","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-d7vcr not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:30.106Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"4508c7c3-bfdc-43e6-a1b1-8216249f5e1e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-d7vcr not found"}
{"level":"info","ts":"2026-06-05T09:39:30.106Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"4508c7c3-bfdc-43e6-a1b1-8216249f5e1e","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"fail-d7vcr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:30.106Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"4508c7c3-bfdc-43e6-a1b1-8216249f5e1e","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"fail-d7vcr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:30.106Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"4508c7c3-bfdc-43e6-a1b1-8216249f5e1e","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"fail-d7vcr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:30.106Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"4508c7c3-bfdc-43e6-a1b1-8216249f5e1e","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-05T09:39:30.106Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"f20c0f6e-ac23-401b-a6d7-f27b87cfb518","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:30.106Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"f20c0f6e-ac23-401b-a6d7-f27b87cfb518","name":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:30.106Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"4508c7c3-bfdc-43e6-a1b1-8216249f5e1e","name":"fail-d7vcr","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-d7vcr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:39:30.106Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"f20c0f6e-ac23-401b-a6d7-f27b87cfb518","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:30.106Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"f20c0f6e-ac23-401b-a6d7-f27b87cfb518"}
{"level":"info","ts":"2026-06-05T09:39:30.106Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"f20c0f6e-ac23-401b-a6d7-f27b87cfb518","RayCluster":"fail-d7vcr","State":""}
{"level":"info","ts":"2026-06-05T09:39:30.112Z","logger":"raycluster-resource","msg":"default","name":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:30.112Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-d7vcr","namespace":"test-ns-l4cdh"}
{"level":"info","ts":"2026-06-05T09:39:30.113Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"4508c7c3-bfdc-43e6-a1b1-8216249f5e1e","name":"fail-d7vcr","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-d7vcr\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-l4cdh","name":"fail-d7vcr"}}
{"level":"info","ts":"2026-06-05T09:39:30.113Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"4508c7c3-bfdc-43e6-a1b1-8216249f5e1e"}
{"level":"error","ts":"2026-06-05T09:39:30.113Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"4508c7c3-bfdc-43e6-a1b1-8216249f5e1e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-d7vcr not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:30.113Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"81f6f589-2657-4d6f-8001-b778d8a07e25"}
{"level":"info","ts":"2026-06-05T09:39:30.113Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"81f6f589-2657-4d6f-8001-b778d8a07e25"}
{"level":"info","ts":"2026-06-05T09:39:30.113Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"81f6f589-2657-4d6f-8001-b778d8a07e25","1 head service found":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:30.113Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"81f6f589-2657-4d6f-8001-b778d8a07e25","error":"head certificate secret ray-head-secret-fail-d7vcr not found"}
{"level":"error","ts":"2026-06-05T09:39:30.113Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"81f6f589-2657-4d6f-8001-b778d8a07e25","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-d7vcr not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:30.113Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"81f6f589-2657-4d6f-8001-b778d8a07e25","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-d7vcr not found"}
{"level":"info","ts":"2026-06-05T09:39:30.113Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"81f6f589-2657-4d6f-8001-b778d8a07e25","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"fail-d7vcr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:30.113Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"81f6f589-2657-4d6f-8001-b778d8a07e25","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"fail-d7vcr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:30.113Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"81f6f589-2657-4d6f-8001-b778d8a07e25","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"fail-d7vcr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:30.113Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"81f6f589-2657-4d6f-8001-b778d8a07e25"}
{"level":"error","ts":"2026-06-05T09:39:30.113Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"81f6f589-2657-4d6f-8001-b778d8a07e25","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-d7vcr not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:30.117Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"e48cafc6-c92f-4363-a36e-9b5e0ca96a08","name":"fail-d7vcr-workers"}
{"level":"info","ts":"2026-06-05T09:39:30.117Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"e48cafc6-c92f-4363-a36e-9b5e0ca96a08","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:30.117Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"516273fb-cb8c-4dc7-bd4a-1453badf643c","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:30.118Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"915ba1dc-689d-498e-af4d-1a5bfceaf12e"}
{"level":"info","ts":"2026-06-05T09:39:30.118Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"915ba1dc-689d-498e-af4d-1a5bfceaf12e","rayCluster":"fail-d7vcr","podIPs":[]}
{"level":"info","ts":"2026-06-05T09:39:30.205Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"717d0a83-bab4-443e-bd90-bdd37dc014da"}
{"level":"info","ts":"2026-06-05T09:39:30.206Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"717d0a83-bab4-443e-bd90-bdd37dc014da"}
{"level":"info","ts":"2026-06-05T09:39:30.206Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"717d0a83-bab4-443e-bd90-bdd37dc014da","1 head service found":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:30.206Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"717d0a83-bab4-443e-bd90-bdd37dc014da","error":"head certificate secret ray-head-secret-fail-d7vcr not found"}
{"level":"error","ts":"2026-06-05T09:39:30.206Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"dab3a8d9-67e8-408e-aec3-e382eef73c34","error":"failed to add finalizer: Operation cannot be fulfilled on rayclusters.ray.io \"fail-d7vcr\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-05T09:39:30.206Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"717d0a83-bab4-443e-bd90-bdd37dc014da","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-d7vcr not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:30.206Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"717d0a83-bab4-443e-bd90-bdd37dc014da","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-d7vcr not found"}
{"level":"info","ts":"2026-06-05T09:39:30.206Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"dab3a8d9-67e8-408e-aec3-e382eef73c34"}
{"level":"error","ts":"2026-06-05T09:39:30.206Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"dab3a8d9-67e8-408e-aec3-e382eef73c34","error":"failed to add finalizer: Operation cannot be fulfilled on rayclusters.ray.io \"fail-d7vcr\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:30.206Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"717d0a83-bab4-443e-bd90-bdd37dc014da","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"fail-d7vcr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:30.206Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"f8aa7b6d-cfe1-4b45-a78d-a64477526f24","namespacedName":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:30.206Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"717d0a83-bab4-443e-bd90-bdd37dc014da","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"fail-d7vcr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:30.206Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"f8aa7b6d-cfe1-4b45-a78d-a64477526f24","mode":"IntegratedOAuth","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:30.206Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"f8aa7b6d-cfe1-4b45-a78d-a64477526f24","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:30.206Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"717d0a83-bab4-443e-bd90-bdd37dc014da","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"fail-d7vcr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:30.206Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"f8aa7b6d-cfe1-4b45-a78d-a64477526f24","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:30.206Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"717d0a83-bab4-443e-bd90-bdd37dc014da"}
{"level":"error","ts":"2026-06-05T09:39:30.206Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"717d0a83-bab4-443e-bd90-bdd37dc014da","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-d7vcr not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:30.211Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"516273fb-cb8c-4dc7-bd4a-1453badf643c","name":"fail-d7vcr-head"}
{"level":"info","ts":"2026-06-05T09:39:30.212Z","logger":"raycluster-resource","msg":"default","name":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:30.212Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-d7vcr","namespace":"test-ns-l4cdh"}
{"level":"info","ts":"2026-06-05T09:39:30.217Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"915ba1dc-689d-498e-af4d-1a5bfceaf12e"}
{"level":"info","ts":"2026-06-05T09:39:30.217Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"915ba1dc-689d-498e-af4d-1a5bfceaf12e","rayCluster":"fail-d7vcr","podIPs":[]}
{"level":"info","ts":"2026-06-05T09:39:30.217Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"915ba1dc-689d-498e-af4d-1a5bfceaf12e"}
{"level":"info","ts":"2026-06-05T09:39:30.218Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"df6bf3cb-d132-4a80-88d8-ff7593bba42f","rayCluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:30.218Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"df6bf3cb-d132-4a80-88d8-ff7593bba42f"}
{"level":"info","ts":"2026-06-05T09:39:30.223Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"c7e15f2c-2231-4842-b0e0-151c96fdb6a5","rayCluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:30.223Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"c7e15f2c-2231-4842-b0e0-151c96fdb6a5"}
{"level":"info","ts":"2026-06-05T09:39:30.223Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"684500b1-06c9-4aa5-bae2-65b71cf90c0a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:30.223Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"684500b1-06c9-4aa5-bae2-65b71cf90c0a","name":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:30.223Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"684500b1-06c9-4aa5-bae2-65b71cf90c0a","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:30.224Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"684500b1-06c9-4aa5-bae2-65b71cf90c0a"}
{"level":"info","ts":"2026-06-05T09:39:30.224Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"684500b1-06c9-4aa5-bae2-65b71cf90c0a","RayCluster":"fail-d7vcr","State":""}
{"level":"info","ts":"2026-06-05T09:39:30.225Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"516273fb-cb8c-4dc7-bd4a-1453badf643c","name":"fail-d7vcr-workers"}
{"level":"info","ts":"2026-06-05T09:39:30.225Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"516273fb-cb8c-4dc7-bd4a-1453badf643c","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:30.225Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"8d1a711e-a3a1-43ac-9efc-5f7ea9af7ca8","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:30.232Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"f8aa7b6d-cfe1-4b45-a78d-a64477526f24","name":"fail-d7vcr-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:39:30.237Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"f8aa7b6d-cfe1-4b45-a78d-a64477526f24","name":"kuberay-gateway-access","namespace":"test-ns-l4cdh","operation":"created"}
{"level":"info","ts":"2026-06-05T09:39:30.238Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"8d1a711e-a3a1-43ac-9efc-5f7ea9af7ca8","name":"fail-d7vcr-head"}
{"level":"info","ts":"2026-06-05T09:39:30.306Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"5616c6f7-9660-4fcb-be16-fcb845fcc444"}
{"level":"info","ts":"2026-06-05T09:39:30.306Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"5616c6f7-9660-4fcb-be16-fcb845fcc444"}
{"level":"info","ts":"2026-06-05T09:39:30.306Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"5616c6f7-9660-4fcb-be16-fcb845fcc444","1 head service found":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:30.306Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"5616c6f7-9660-4fcb-be16-fcb845fcc444","error":"head certificate secret ray-head-secret-fail-d7vcr not found"}
{"level":"info","ts":"2026-06-05T09:39:30.306Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"f8aa7b6d-cfe1-4b45-a78d-a64477526f24","name":"test-ns-l4cdh-fail-d7vcr","operation":"created"}
{"level":"error","ts":"2026-06-05T09:39:30.306Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"5616c6f7-9660-4fcb-be16-fcb845fcc444","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-d7vcr not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:30.306Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"5616c6f7-9660-4fcb-be16-fcb845fcc444","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-d7vcr not found"}
{"level":"info","ts":"2026-06-05T09:39:30.306Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"5616c6f7-9660-4fcb-be16-fcb845fcc444","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"fail-d7vcr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:30.306Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"5616c6f7-9660-4fcb-be16-fcb845fcc444","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"fail-d7vcr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:30.306Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"5616c6f7-9660-4fcb-be16-fcb845fcc444","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"fail-d7vcr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:30.306Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"5616c6f7-9660-4fcb-be16-fcb845fcc444"}
{"level":"error","ts":"2026-06-05T09:39:30.306Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"5616c6f7-9660-4fcb-be16-fcb845fcc444","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-d7vcr not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:30.310Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"8d1a711e-a3a1-43ac-9efc-5f7ea9af7ca8","name":"fail-d7vcr-workers"}
{"level":"info","ts":"2026-06-05T09:39:30.310Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"8d1a711e-a3a1-43ac-9efc-5f7ea9af7ca8","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:30.311Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"f8aa7b6d-cfe1-4b45-a78d-a64477526f24","name":"kube-rbac-proxy-config-fail-d7vcr","operation":"created"}
{"level":"error","ts":"2026-06-05T09:39:30.318Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"f8aa7b6d-cfe1-4b45-a78d-a64477526f24","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-d7vcr\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:30.319Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"f8aa7b6d-cfe1-4b45-a78d-a64477526f24"}
{"level":"error","ts":"2026-06-05T09:39:30.319Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"f8aa7b6d-cfe1-4b45-a78d-a64477526f24","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-d7vcr\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:30.319Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"bededb69-2ebd-4803-bd5b-8d329896eee0","namespacedName":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:30.319Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"bededb69-2ebd-4803-bd5b-8d329896eee0","mode":"IntegratedOAuth","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:30.319Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"bededb69-2ebd-4803-bd5b-8d329896eee0","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:30.331Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"bededb69-2ebd-4803-bd5b-8d329896eee0","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:30.331Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"c8175b8c-d4a8-4cb6-b00c-c7e7ad982d68","rayCluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:30.331Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"2b42081f-048c-4e96-9bd1-e42c9a0208a2","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:30.331Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"c8175b8c-d4a8-4cb6-b00c-c7e7ad982d68"}
{"level":"info","ts":"2026-06-05T09:39:30.331Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"2809d259-794d-4502-9894-d035f333b3a1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:30.331Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"2809d259-794d-4502-9894-d035f333b3a1","name":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:30.331Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"2809d259-794d-4502-9894-d035f333b3a1","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:30.331Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"2809d259-794d-4502-9894-d035f333b3a1"}
{"level":"info","ts":"2026-06-05T09:39:30.331Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"2809d259-794d-4502-9894-d035f333b3a1","RayCluster":"fail-d7vcr","State":""}
{"level":"info","ts":"2026-06-05T09:39:30.339Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"893d4c4a-1f11-4ae3-844f-ff9f366c6eea","namespacedName":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:30.339Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"893d4c4a-1f11-4ae3-844f-ff9f366c6eea","mode":"IntegratedOAuth","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:30.339Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"893d4c4a-1f11-4ae3-844f-ff9f366c6eea","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:30.341Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"2b42081f-048c-4e96-9bd1-e42c9a0208a2","name":"fail-d7vcr-head"}
{"level":"info","ts":"2026-06-05T09:39:30.405Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"b8796136-e7cc-4d5c-8335-b45f6e2db173"}
{"level":"info","ts":"2026-06-05T09:39:30.405Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"b8796136-e7cc-4d5c-8335-b45f6e2db173"}
{"level":"info","ts":"2026-06-05T09:39:30.406Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"b8796136-e7cc-4d5c-8335-b45f6e2db173","1 head service found":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:30.406Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"b8796136-e7cc-4d5c-8335-b45f6e2db173","error":"head certificate secret ray-head-secret-fail-d7vcr not found"}
{"level":"error","ts":"2026-06-05T09:39:30.406Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"b8796136-e7cc-4d5c-8335-b45f6e2db173","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-d7vcr not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:30.406Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"b8796136-e7cc-4d5c-8335-b45f6e2db173","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-d7vcr not found"}
{"level":"info","ts":"2026-06-05T09:39:30.406Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"b8796136-e7cc-4d5c-8335-b45f6e2db173","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"fail-d7vcr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:30.406Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"b8796136-e7cc-4d5c-8335-b45f6e2db173","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"fail-d7vcr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:30.406Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"b8796136-e7cc-4d5c-8335-b45f6e2db173","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"fail-d7vcr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:30.406Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"b8796136-e7cc-4d5c-8335-b45f6e2db173"}
{"level":"error","ts":"2026-06-05T09:39:30.406Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"b8796136-e7cc-4d5c-8335-b45f6e2db173","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-d7vcr not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:30.406Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"893d4c4a-1f11-4ae3-844f-ff9f366c6eea","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:30.409Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"2b42081f-048c-4e96-9bd1-e42c9a0208a2","name":"fail-d7vcr-workers"}
{"level":"info","ts":"2026-06-05T09:39:30.410Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"2b42081f-048c-4e96-9bd1-e42c9a0208a2","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:30.566Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"4ff1b21c-1b1f-40b9-9822-30140a086fdb"}
{"level":"info","ts":"2026-06-05T09:39:30.566Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"4ff1b21c-1b1f-40b9-9822-30140a086fdb"}
{"level":"info","ts":"2026-06-05T09:39:30.566Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"4ff1b21c-1b1f-40b9-9822-30140a086fdb","1 head service found":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:30.567Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"4ff1b21c-1b1f-40b9-9822-30140a086fdb","error":"head certificate secret ray-head-secret-fail-d7vcr not found"}
{"level":"error","ts":"2026-06-05T09:39:30.567Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"4ff1b21c-1b1f-40b9-9822-30140a086fdb","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-d7vcr not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:30.567Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"4ff1b21c-1b1f-40b9-9822-30140a086fdb","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-d7vcr not found"}
{"level":"info","ts":"2026-06-05T09:39:30.567Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"4ff1b21c-1b1f-40b9-9822-30140a086fdb","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"fail-d7vcr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:30.567Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"4ff1b21c-1b1f-40b9-9822-30140a086fdb","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"fail-d7vcr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:30.567Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"4ff1b21c-1b1f-40b9-9822-30140a086fdb","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"fail-d7vcr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:30.567Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"4ff1b21c-1b1f-40b9-9822-30140a086fdb"}
{"level":"error","ts":"2026-06-05T09:39:30.567Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"4ff1b21c-1b1f-40b9-9822-30140a086fdb","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-d7vcr not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:30.887Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"62f330ea-912b-41dd-830e-4304365f3cf9"}
{"level":"info","ts":"2026-06-05T09:39:30.887Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"62f330ea-912b-41dd-830e-4304365f3cf9"}
{"level":"info","ts":"2026-06-05T09:39:30.887Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"62f330ea-912b-41dd-830e-4304365f3cf9","1 head service found":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:30.887Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"62f330ea-912b-41dd-830e-4304365f3cf9","error":"head certificate secret ray-head-secret-fail-d7vcr not found"}
{"level":"error","ts":"2026-06-05T09:39:30.887Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"62f330ea-912b-41dd-830e-4304365f3cf9","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-d7vcr not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:30.888Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"62f330ea-912b-41dd-830e-4304365f3cf9","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-d7vcr not found"}
{"level":"info","ts":"2026-06-05T09:39:30.888Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"62f330ea-912b-41dd-830e-4304365f3cf9","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"fail-d7vcr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:30.888Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"62f330ea-912b-41dd-830e-4304365f3cf9","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"fail-d7vcr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:30.888Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"62f330ea-912b-41dd-830e-4304365f3cf9","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"fail-d7vcr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:30.888Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"62f330ea-912b-41dd-830e-4304365f3cf9"}
{"level":"error","ts":"2026-06-05T09:39:30.888Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"62f330ea-912b-41dd-830e-4304365f3cf9","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-d7vcr not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:31.528Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"da8dfaa7-391c-45b1-b032-52f300b34748"}
{"level":"info","ts":"2026-06-05T09:39:31.528Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"da8dfaa7-391c-45b1-b032-52f300b34748"}
{"level":"info","ts":"2026-06-05T09:39:31.528Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"da8dfaa7-391c-45b1-b032-52f300b34748","1 head service found":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:31.528Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"da8dfaa7-391c-45b1-b032-52f300b34748","error":"head certificate secret ray-head-secret-fail-d7vcr not found"}
{"level":"error","ts":"2026-06-05T09:39:31.528Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"da8dfaa7-391c-45b1-b032-52f300b34748","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-d7vcr not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:31.529Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"da8dfaa7-391c-45b1-b032-52f300b34748","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-d7vcr not found"}
{"level":"info","ts":"2026-06-05T09:39:31.529Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"da8dfaa7-391c-45b1-b032-52f300b34748","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"fail-d7vcr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:31.529Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"da8dfaa7-391c-45b1-b032-52f300b34748","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"fail-d7vcr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:31.529Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"da8dfaa7-391c-45b1-b032-52f300b34748","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"fail-d7vcr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:39:31.529Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"da8dfaa7-391c-45b1-b032-52f300b34748"}
{"level":"error","ts":"2026-06-05T09:39:31.529Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"da8dfaa7-391c-45b1-b032-52f300b34748","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-d7vcr not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:32.793Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"230df625-0325-405a-ba12-0c0560e5753e","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:32.793Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"230df625-0325-405a-ba12-0c0560e5753e","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-05 09:39:29 +0000 UTC","Now":"2026-06-05T09:39:32.793Z","ShutdownTime":"2026-06-05T09:39:29.000Z"}
{"level":"info","ts":"2026-06-05T09:39:32.794Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"230df625-0325-405a-ba12-0c0560e5753e","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-05T09:39:32.794Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"230df625-0325-405a-ba12-0c0560e5753e","RayCluster":{"name":"counter-85b5h","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:32.794Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"230df625-0325-405a-ba12-0c0560e5753e","isClusterDeleted":true}
{"level":"info","ts":"2026-06-05T09:39:32.794Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"230df625-0325-405a-ba12-0c0560e5753e","RayCluster":"counter-85b5h"}
{"level":"info","ts":"2026-06-05T09:39:32.810Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"79e67902-3ed9-469b-bcc4-d60bcbbcbe13"}
{"level":"info","ts":"2026-06-05T09:39:32.810Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"79e67902-3ed9-469b-bcc4-d60bcbbcbe13"}
{"level":"info","ts":"2026-06-05T09:39:32.810Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"79e67902-3ed9-469b-bcc4-d60bcbbcbe13","1 head service found":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:32.810Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"79e67902-3ed9-469b-bcc4-d60bcbbcbe13"}
{"level":"info","ts":"2026-06-05T09:39:32.810Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"79e67902-3ed9-469b-bcc4-d60bcbbcbe13","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:39:32.810Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"79e67902-3ed9-469b-bcc4-d60bcbbcbe13"}
{"level":"info","ts":"2026-06-05T09:39:32.810Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"79e67902-3ed9-469b-bcc4-d60bcbbcbe13","mode":"IntegratedOAuth","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:32.810Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"79e67902-3ed9-469b-bcc4-d60bcbbcbe13","cluster":"fail-d7vcr","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:39:32.810Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"79e67902-3ed9-469b-bcc4-d60bcbbcbe13","cluster":"fail-d7vcr","authType":"IntegratedOAuth","serviceAccount":"fail-d7vcr-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-05T09:39:32.810Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"79e67902-3ed9-469b-bcc4-d60bcbbcbe13","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"fail-d7vcr","ray.io/group":"headgroup","ray.io/identifier":"fail-d7vcr-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-05T09:39:32.810Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"79e67902-3ed9-469b-bcc4-d60bcbbcbe13","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-05T09:39:32.810Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"79e67902-3ed9-469b-bcc4-d60bcbbcbe13","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-05T09:39:32.810Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"79e67902-3ed9-469b-bcc4-d60bcbbcbe13","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-05T09:39:32.810Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"79e67902-3ed9-469b-bcc4-d60bcbbcbe13","enabled":true}
{"level":"info","ts":"2026-06-05T09:39:32.827Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"79e67902-3ed9-469b-bcc4-d60bcbbcbe13","name":"fail-d7vcr-head-8br8s"}
{"level":"info","ts":"2026-06-05T09:39:32.827Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"79e67902-3ed9-469b-bcc4-d60bcbbcbe13","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:39:32.827Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"79e67902-3ed9-469b-bcc4-d60bcbbcbe13","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:39:32.827Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"79e67902-3ed9-469b-bcc4-d60bcbbcbe13","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-05T09:39:32.827Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"79e67902-3ed9-469b-bcc4-d60bcbbcbe13","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-05T09:39:32.827Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"79e67902-3ed9-469b-bcc4-d60bcbbcbe13","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-05T09:39:32.827Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"79e67902-3ed9-469b-bcc4-d60bcbbcbe13"}
{"level":"info","ts":"2026-06-05T09:39:32.827Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"79e67902-3ed9-469b-bcc4-d60bcbbcbe13","nodeType":"worker","rayStartParams":{"address":"fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-05T09:39:32.827Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"79e67902-3ed9-469b-bcc4-d60bcbbcbe13","rayStartCmd":"ray start  --address=fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:39:32.827Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"79e67902-3ed9-469b-bcc4-d60bcbbcbe13","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:39:32.827Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"79e67902-3ed9-469b-bcc4-d60bcbbcbe13","enabled":true}
{"level":"info","ts":"2026-06-05T09:39:32.843Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"79e67902-3ed9-469b-bcc4-d60bcbbcbe13","name":"fail-d7vcr-small-group-worker-crtnw"}
{"level":"info","ts":"2026-06-05T09:39:32.843Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"79e67902-3ed9-469b-bcc4-d60bcbbcbe13","podName":"fail-d7vcr-head-8br8s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:39:32.843Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"79e67902-3ed9-469b-bcc4-d60bcbbcbe13","podName":"fail-d7vcr-head-8br8s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:39:32.843Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"79e67902-3ed9-469b-bcc4-d60bcbbcbe13","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"fail-d7vcr-head-svc"},"newHead":{"podName":"fail-d7vcr-head-8br8s","serviceName":"fail-d7vcr-head-svc"}}
{"level":"info","ts":"2026-06-05T09:39:32.843Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"79e67902-3ed9-469b-bcc4-d60bcbbcbe13","name":"fail-d7vcr","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-d7vcr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-d7vcr-head-8br8s","serviceName":"fail-d7vcr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:39:32.859Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"ab788284-35e3-45e7-a2ed-0cf5d3c2e83e"}
{"level":"info","ts":"2026-06-05T09:39:32.859Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"ab788284-35e3-45e7-a2ed-0cf5d3c2e83e"}
{"level":"info","ts":"2026-06-05T09:39:32.859Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"ab788284-35e3-45e7-a2ed-0cf5d3c2e83e","1 head service found":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:32.859Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"ab788284-35e3-45e7-a2ed-0cf5d3c2e83e","Found 1 head Pod":"fail-d7vcr-head-8br8s","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:39:32.859Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"ab788284-35e3-45e7-a2ed-0cf5d3c2e83e","head Pod":"fail-d7vcr-head-8br8s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-d7vcr-head-8br8s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:32.859Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"ab788284-35e3-45e7-a2ed-0cf5d3c2e83e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:39:32.859Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"ab788284-35e3-45e7-a2ed-0cf5d3c2e83e","worker Pod":"fail-d7vcr-small-group-worker-crtnw","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-d7vcr-small-group-worker-crtnw. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:32.859Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"ab788284-35e3-45e7-a2ed-0cf5d3c2e83e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:39:32.859Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"ab788284-35e3-45e7-a2ed-0cf5d3c2e83e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:39:32.859Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"ab788284-35e3-45e7-a2ed-0cf5d3c2e83e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:39:32.860Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"b69ef0ef-7f70-4992-9b18-8573793ab8c5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:32.860Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"b69ef0ef-7f70-4992-9b18-8573793ab8c5","name":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:32.860Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"847fda05-4b62-4569-bac2-434bab3d21ff","rayCluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:32.860Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"ab788284-35e3-45e7-a2ed-0cf5d3c2e83e","podName":"fail-d7vcr-small-group-worker-crtnw","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:39:32.860Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"b69ef0ef-7f70-4992-9b18-8573793ab8c5","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:32.860Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"ab788284-35e3-45e7-a2ed-0cf5d3c2e83e","podName":"fail-d7vcr-small-group-worker-crtnw","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:39:32.860Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"847fda05-4b62-4569-bac2-434bab3d21ff","rayCluster":"fail-d7vcr","podIPs":[]}
{"level":"info","ts":"2026-06-05T09:39:32.860Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"ab788284-35e3-45e7-a2ed-0cf5d3c2e83e","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"fail-d7vcr-head-svc"},"newHead":{"podName":"fail-d7vcr-head-8br8s","serviceName":"fail-d7vcr-head-svc"}}
{"level":"info","ts":"2026-06-05T09:39:32.860Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"b69ef0ef-7f70-4992-9b18-8573793ab8c5"}
{"level":"info","ts":"2026-06-05T09:39:32.860Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"b69ef0ef-7f70-4992-9b18-8573793ab8c5","RayCluster":"fail-d7vcr","State":""}
{"level":"info","ts":"2026-06-05T09:39:32.860Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"ab788284-35e3-45e7-a2ed-0cf5d3c2e83e","name":"fail-d7vcr","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-d7vcr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-d7vcr-head-8br8s","serviceName":"fail-d7vcr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:39:32.868Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"ab788284-35e3-45e7-a2ed-0cf5d3c2e83e","name":"fail-d7vcr","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-d7vcr\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-l4cdh","name":"fail-d7vcr"}}
{"level":"info","ts":"2026-06-05T09:39:32.868Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"ab788284-35e3-45e7-a2ed-0cf5d3c2e83e"}
{"level":"error","ts":"2026-06-05T09:39:32.868Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"ab788284-35e3-45e7-a2ed-0cf5d3c2e83e","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-d7vcr\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:32.868Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"9f447e23-79c4-4703-a0a5-f8912d1b1e37"}
{"level":"info","ts":"2026-06-05T09:39:32.868Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"9f447e23-79c4-4703-a0a5-f8912d1b1e37"}
{"level":"info","ts":"2026-06-05T09:39:32.868Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"9f447e23-79c4-4703-a0a5-f8912d1b1e37","1 head service found":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:32.868Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"9f447e23-79c4-4703-a0a5-f8912d1b1e37","Found 1 head Pod":"fail-d7vcr-head-8br8s","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:39:32.868Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"9f447e23-79c4-4703-a0a5-f8912d1b1e37","head Pod":"fail-d7vcr-head-8br8s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-d7vcr-head-8br8s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:32.868Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"9f447e23-79c4-4703-a0a5-f8912d1b1e37","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:39:32.868Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"9f447e23-79c4-4703-a0a5-f8912d1b1e37","worker Pod":"fail-d7vcr-small-group-worker-crtnw","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-d7vcr-small-group-worker-crtnw. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:32.868Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"9f447e23-79c4-4703-a0a5-f8912d1b1e37","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:39:32.868Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"9f447e23-79c4-4703-a0a5-f8912d1b1e37","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:39:32.868Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"9f447e23-79c4-4703-a0a5-f8912d1b1e37","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:39:32.868Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"9f447e23-79c4-4703-a0a5-f8912d1b1e37","podName":"fail-d7vcr-head-8br8s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:39:32.868Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"9f447e23-79c4-4703-a0a5-f8912d1b1e37","podName":"fail-d7vcr-head-8br8s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:39:32.869Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"9f447e23-79c4-4703-a0a5-f8912d1b1e37","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:39:32.869Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"9f447e23-79c4-4703-a0a5-f8912d1b1e37","seconds":300}
{"level":"info","ts":"2026-06-05T09:39:32.874Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"1d5aecf8-401d-4dbb-a8f8-a968f965d2db"}
{"level":"info","ts":"2026-06-05T09:39:32.874Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"1d5aecf8-401d-4dbb-a8f8-a968f965d2db"}
{"level":"info","ts":"2026-06-05T09:39:32.874Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"1d5aecf8-401d-4dbb-a8f8-a968f965d2db","1 head service found":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:32.874Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"1d5aecf8-401d-4dbb-a8f8-a968f965d2db","Found 1 head Pod":"fail-d7vcr-head-8br8s","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:39:32.874Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"1d5aecf8-401d-4dbb-a8f8-a968f965d2db","head Pod":"fail-d7vcr-head-8br8s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-d7vcr-head-8br8s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:32.874Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"1d5aecf8-401d-4dbb-a8f8-a968f965d2db","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:39:32.874Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"1d5aecf8-401d-4dbb-a8f8-a968f965d2db","worker Pod":"fail-d7vcr-small-group-worker-crtnw","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-d7vcr-small-group-worker-crtnw. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:32.874Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"1d5aecf8-401d-4dbb-a8f8-a968f965d2db","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:39:32.874Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"1d5aecf8-401d-4dbb-a8f8-a968f965d2db","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:39:32.874Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"1d5aecf8-401d-4dbb-a8f8-a968f965d2db","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:39:32.874Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"1d5aecf8-401d-4dbb-a8f8-a968f965d2db","podName":"fail-d7vcr-head-8br8s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:39:32.874Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"1d5aecf8-401d-4dbb-a8f8-a968f965d2db","podName":"fail-d7vcr-head-8br8s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:39:32.874Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"1d5aecf8-401d-4dbb-a8f8-a968f965d2db","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:39:32.874Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"1d5aecf8-401d-4dbb-a8f8-a968f965d2db","seconds":300}
{"level":"info","ts":"2026-06-05T09:39:32.905Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"67256d46-db85-409b-ab6b-102292145e6c","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:32.906Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c35ed462-5c4c-46be-9b51-eb838f3cb648"}
{"level":"info","ts":"2026-06-05T09:39:32.906Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c35ed462-5c4c-46be-9b51-eb838f3cb648"}
{"level":"info","ts":"2026-06-05T09:39:32.906Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c35ed462-5c4c-46be-9b51-eb838f3cb648","1 head service found":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:32.906Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c35ed462-5c4c-46be-9b51-eb838f3cb648","Found 1 head Pod":"fail-d7vcr-head-8br8s","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:39:32.906Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c35ed462-5c4c-46be-9b51-eb838f3cb648","head Pod":"fail-d7vcr-head-8br8s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-d7vcr-head-8br8s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:32.906Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c35ed462-5c4c-46be-9b51-eb838f3cb648","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:39:32.906Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c35ed462-5c4c-46be-9b51-eb838f3cb648","worker Pod":"fail-d7vcr-small-group-worker-crtnw","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-d7vcr-small-group-worker-crtnw. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:32.906Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c35ed462-5c4c-46be-9b51-eb838f3cb648","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:39:32.906Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c35ed462-5c4c-46be-9b51-eb838f3cb648","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:39:32.906Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c35ed462-5c4c-46be-9b51-eb838f3cb648","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:39:32.906Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c35ed462-5c4c-46be-9b51-eb838f3cb648","podName":"fail-d7vcr-head-8br8s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:39:32.906Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c35ed462-5c4c-46be-9b51-eb838f3cb648","podName":"fail-d7vcr-head-8br8s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:39:32.906Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c35ed462-5c4c-46be-9b51-eb838f3cb648","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-05T09:39:32.906Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c35ed462-5c4c-46be-9b51-eb838f3cb648","name":"fail-d7vcr","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-d7vcr-head-8br8s","serviceName":"fail-d7vcr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-d7vcr-head-8br8s","serviceName":"fail-d7vcr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:39:32.917Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"67256d46-db85-409b-ab6b-102292145e6c","name":"fail-d7vcr-head"}
{"level":"info","ts":"2026-06-05T09:39:32.918Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"5e75845e-6fc6-4c78-b859-adcb2edaa239"}
{"level":"info","ts":"2026-06-05T09:39:32.918Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"5e75845e-6fc6-4c78-b859-adcb2edaa239"}
{"level":"info","ts":"2026-06-05T09:39:32.918Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"5e75845e-6fc6-4c78-b859-adcb2edaa239","1 head service found":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:32.918Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"5e75845e-6fc6-4c78-b859-adcb2edaa239","Found 1 head Pod":"fail-d7vcr-head-8br8s","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:39:32.918Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"5e75845e-6fc6-4c78-b859-adcb2edaa239","head Pod":"fail-d7vcr-head-8br8s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-d7vcr-head-8br8s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:32.918Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"5e75845e-6fc6-4c78-b859-adcb2edaa239","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:39:32.918Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"5e75845e-6fc6-4c78-b859-adcb2edaa239","worker Pod":"fail-d7vcr-small-group-worker-crtnw","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-d7vcr-small-group-worker-crtnw. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:32.918Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"5e75845e-6fc6-4c78-b859-adcb2edaa239","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:39:32.918Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"5e75845e-6fc6-4c78-b859-adcb2edaa239","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:39:32.918Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"5e75845e-6fc6-4c78-b859-adcb2edaa239","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:39:32.918Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"5e75845e-6fc6-4c78-b859-adcb2edaa239","podName":"fail-d7vcr-head-8br8s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:39:32.918Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"5e75845e-6fc6-4c78-b859-adcb2edaa239","podName":"fail-d7vcr-head-8br8s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:39:32.918Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"5e75845e-6fc6-4c78-b859-adcb2edaa239","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-05T09:39:32.918Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"5e75845e-6fc6-4c78-b859-adcb2edaa239","name":"fail-d7vcr","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-d7vcr-head-8br8s","serviceName":"fail-d7vcr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-d7vcr-head-8br8s","serviceName":"fail-d7vcr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:39:32.918Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"a7ae993e-1aed-4dd6-86a1-cd8629726fd0","rayCluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:32.918Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"a7ae993e-1aed-4dd6-86a1-cd8629726fd0","rayCluster":"fail-d7vcr","podIPs":[]}
{"level":"info","ts":"2026-06-05T09:39:32.918Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"c463b9f3-8822-4e84-bcb0-e30e0be42979","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:32.918Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"c463b9f3-8822-4e84-bcb0-e30e0be42979","name":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:32.919Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"c463b9f3-8822-4e84-bcb0-e30e0be42979","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:32.919Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"c463b9f3-8822-4e84-bcb0-e30e0be42979"}
{"level":"info","ts":"2026-06-05T09:39:32.919Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"c463b9f3-8822-4e84-bcb0-e30e0be42979","RayCluster":"fail-d7vcr","State":""}
{"level":"info","ts":"2026-06-05T09:39:32.925Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"5e75845e-6fc6-4c78-b859-adcb2edaa239","name":"fail-d7vcr","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-d7vcr\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-l4cdh","name":"fail-d7vcr"}}
{"level":"info","ts":"2026-06-05T09:39:32.925Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"5e75845e-6fc6-4c78-b859-adcb2edaa239"}
{"level":"error","ts":"2026-06-05T09:39:32.925Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"5e75845e-6fc6-4c78-b859-adcb2edaa239","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-d7vcr\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:39:32.929Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"67256d46-db85-409b-ab6b-102292145e6c","name":"fail-d7vcr-workers"}
{"level":"info","ts":"2026-06-05T09:39:32.929Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"67256d46-db85-409b-ab6b-102292145e6c","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:32.929Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"168c8560-b76f-49bc-b6ca-ab679f2e73a5","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:32.931Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"b28afe9b-9293-4fc6-b94d-cfae3a5b93ca"}
{"level":"info","ts":"2026-06-05T09:39:32.931Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"b28afe9b-9293-4fc6-b94d-cfae3a5b93ca"}
{"level":"info","ts":"2026-06-05T09:39:32.931Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"b28afe9b-9293-4fc6-b94d-cfae3a5b93ca","1 head service found":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:32.931Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"b28afe9b-9293-4fc6-b94d-cfae3a5b93ca","Found 1 head Pod":"fail-d7vcr-head-8br8s","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:39:32.931Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"b28afe9b-9293-4fc6-b94d-cfae3a5b93ca","head Pod":"fail-d7vcr-head-8br8s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-d7vcr-head-8br8s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:32.931Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"b28afe9b-9293-4fc6-b94d-cfae3a5b93ca","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:39:32.931Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"b28afe9b-9293-4fc6-b94d-cfae3a5b93ca","worker Pod":"fail-d7vcr-small-group-worker-crtnw","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-d7vcr-small-group-worker-crtnw. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:32.931Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"b28afe9b-9293-4fc6-b94d-cfae3a5b93ca","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:39:32.931Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"b28afe9b-9293-4fc6-b94d-cfae3a5b93ca","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:39:32.931Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"b28afe9b-9293-4fc6-b94d-cfae3a5b93ca","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:39:32.932Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"b28afe9b-9293-4fc6-b94d-cfae3a5b93ca","podName":"fail-d7vcr-head-8br8s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:39:32.932Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"b28afe9b-9293-4fc6-b94d-cfae3a5b93ca","podName":"fail-d7vcr-head-8br8s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:39:32.932Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"b28afe9b-9293-4fc6-b94d-cfae3a5b93ca","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:39:32.932Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"b28afe9b-9293-4fc6-b94d-cfae3a5b93ca","seconds":300}
{"level":"info","ts":"2026-06-05T09:39:32.942Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"168c8560-b76f-49bc-b6ca-ab679f2e73a5","name":"fail-d7vcr-head"}
{"level":"info","ts":"2026-06-05T09:39:32.951Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"168c8560-b76f-49bc-b6ca-ab679f2e73a5","name":"fail-d7vcr-workers"}
{"level":"info","ts":"2026-06-05T09:39:32.951Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"168c8560-b76f-49bc-b6ca-ab679f2e73a5","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:33.006Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"531836dd-018c-42b0-8d09-98789f5b5dba","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:33.006Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"531836dd-018c-42b0-8d09-98789f5b5dba","name":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:33.006Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"531836dd-018c-42b0-8d09-98789f5b5dba","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:33.006Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"531836dd-018c-42b0-8d09-98789f5b5dba"}
{"level":"info","ts":"2026-06-05T09:39:33.006Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"531836dd-018c-42b0-8d09-98789f5b5dba","RayCluster":"fail-d7vcr","State":""}
{"level":"info","ts":"2026-06-05T09:39:33.281Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c77a65b0-e270-45c5-87fd-e33c5dbfb731"}
{"level":"info","ts":"2026-06-05T09:39:33.281Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c77a65b0-e270-45c5-87fd-e33c5dbfb731"}
{"level":"info","ts":"2026-06-05T09:39:33.281Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c77a65b0-e270-45c5-87fd-e33c5dbfb731","1 head service found":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:33.281Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c77a65b0-e270-45c5-87fd-e33c5dbfb731","Found 1 head Pod":"fail-d7vcr-head-8br8s","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:39:33.281Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c77a65b0-e270-45c5-87fd-e33c5dbfb731","head Pod":"fail-d7vcr-head-8br8s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-d7vcr-head-8br8s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:33.281Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c77a65b0-e270-45c5-87fd-e33c5dbfb731","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:39:33.281Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c77a65b0-e270-45c5-87fd-e33c5dbfb731","worker Pod":"fail-d7vcr-small-group-worker-crtnw","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-d7vcr-small-group-worker-crtnw. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:33.281Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c77a65b0-e270-45c5-87fd-e33c5dbfb731","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:39:33.281Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c77a65b0-e270-45c5-87fd-e33c5dbfb731","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:39:33.281Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c77a65b0-e270-45c5-87fd-e33c5dbfb731","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:39:33.281Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c77a65b0-e270-45c5-87fd-e33c5dbfb731","podName":"fail-d7vcr-small-group-worker-crtnw","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:39:33.282Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c77a65b0-e270-45c5-87fd-e33c5dbfb731","podName":"fail-d7vcr-small-group-worker-crtnw","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:39:33.282Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c77a65b0-e270-45c5-87fd-e33c5dbfb731","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:39:33.282Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c77a65b0-e270-45c5-87fd-e33c5dbfb731","seconds":300}
{"level":"info","ts":"2026-06-05T09:39:33.307Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"d41b56ec-fcc9-4058-8615-53e5c1ab516d"}
{"level":"info","ts":"2026-06-05T09:39:33.308Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"d41b56ec-fcc9-4058-8615-53e5c1ab516d"}
{"level":"info","ts":"2026-06-05T09:39:33.308Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"d41b56ec-fcc9-4058-8615-53e5c1ab516d","1 head service found":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:33.308Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"d41b56ec-fcc9-4058-8615-53e5c1ab516d","Found 1 head Pod":"fail-d7vcr-head-8br8s","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:39:33.308Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"d41b56ec-fcc9-4058-8615-53e5c1ab516d","head Pod":"fail-d7vcr-head-8br8s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-d7vcr-head-8br8s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:33.308Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"d41b56ec-fcc9-4058-8615-53e5c1ab516d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:39:33.308Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"d41b56ec-fcc9-4058-8615-53e5c1ab516d","worker Pod":"fail-d7vcr-small-group-worker-crtnw","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-d7vcr-small-group-worker-crtnw. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:33.308Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"d41b56ec-fcc9-4058-8615-53e5c1ab516d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:39:33.308Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"d41b56ec-fcc9-4058-8615-53e5c1ab516d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:39:33.308Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"d41b56ec-fcc9-4058-8615-53e5c1ab516d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:39:33.308Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"d41b56ec-fcc9-4058-8615-53e5c1ab516d","podName":"fail-d7vcr-small-group-worker-crtnw","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:39:33.308Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"d41b56ec-fcc9-4058-8615-53e5c1ab516d","podName":"fail-d7vcr-small-group-worker-crtnw","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:39:33.308Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"d41b56ec-fcc9-4058-8615-53e5c1ab516d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:39:33.308Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"d41b56ec-fcc9-4058-8615-53e5c1ab516d","seconds":300}
{"level":"info","ts":"2026-06-05T09:39:33.800Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"ab069cae-f5f1-45d8-9316-c5f08da93c9b"}
{"level":"info","ts":"2026-06-05T09:39:33.800Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"ab069cae-f5f1-45d8-9316-c5f08da93c9b"}
{"level":"info","ts":"2026-06-05T09:39:33.800Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"ab069cae-f5f1-45d8-9316-c5f08da93c9b","1 head service found":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:33.800Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"ab069cae-f5f1-45d8-9316-c5f08da93c9b","Found 1 head Pod":"fail-d7vcr-head-8br8s","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:39:33.800Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"ab069cae-f5f1-45d8-9316-c5f08da93c9b","head Pod":"fail-d7vcr-head-8br8s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-d7vcr-head-8br8s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:33.800Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"ab069cae-f5f1-45d8-9316-c5f08da93c9b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:39:33.800Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"ab069cae-f5f1-45d8-9316-c5f08da93c9b","worker Pod":"fail-d7vcr-small-group-worker-crtnw","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-d7vcr-small-group-worker-crtnw. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:33.800Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"ab069cae-f5f1-45d8-9316-c5f08da93c9b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:39:33.800Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"ab069cae-f5f1-45d8-9316-c5f08da93c9b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:39:33.800Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"ab069cae-f5f1-45d8-9316-c5f08da93c9b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:39:33.800Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"ab069cae-f5f1-45d8-9316-c5f08da93c9b","podName":"fail-d7vcr-small-group-worker-crtnw","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:39:33.800Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"ab069cae-f5f1-45d8-9316-c5f08da93c9b","podName":"fail-d7vcr-small-group-worker-crtnw","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:39:33.800Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"ab069cae-f5f1-45d8-9316-c5f08da93c9b","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"fail-d7vcr-head-8br8s","serviceName":"fail-d7vcr-head-svc"},"newHead":{"podIP":"10.132.0.16","serviceIP":"10.132.0.16","podName":"fail-d7vcr-head-8br8s","serviceName":"fail-d7vcr-head-svc"}}
{"level":"info","ts":"2026-06-05T09:39:33.800Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"ab069cae-f5f1-45d8-9316-c5f08da93c9b","name":"fail-d7vcr","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-d7vcr-head-8br8s","serviceName":"fail-d7vcr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.16","serviceIP":"10.132.0.16","podName":"fail-d7vcr-head-8br8s","serviceName":"fail-d7vcr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:39:33.812Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"d6f9833e-ab48-4995-8871-b7c2c23b6da4","rayCluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:33.812Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"21417b63-e278-47f1-9ac8-03e76275f29a","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:33.812Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"d6f9833e-ab48-4995-8871-b7c2c23b6da4","pod":"fail-d7vcr-head-8br8s","ip":"10.132.0.16"}
{"level":"info","ts":"2026-06-05T09:39:33.812Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"d05c842a-8fdc-4389-999c-23c31ab86faa","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:33.812Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"d05c842a-8fdc-4389-999c-23c31ab86faa","name":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:33.812Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"d05c842a-8fdc-4389-999c-23c31ab86faa","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:33.812Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"d05c842a-8fdc-4389-999c-23c31ab86faa"}
{"level":"info","ts":"2026-06-05T09:39:33.812Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"d05c842a-8fdc-4389-999c-23c31ab86faa","RayCluster":"fail-d7vcr","State":""}
{"level":"info","ts":"2026-06-05T09:39:33.824Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"21417b63-e278-47f1-9ac8-03e76275f29a","name":"fail-d7vcr-head"}
{"level":"info","ts":"2026-06-05T09:39:33.831Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"d6f9833e-ab48-4995-8871-b7c2c23b6da4","certificate":"ray-head-cert-fail-d7vcr","dnsNames":["fail-d7vcr-head-svc","localhost","fail-d7vcr-head-svc.test-ns-l4cdh.svc","fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local"],"ipAddresses":["10.132.0.16","127.0.0.1"]}
{"level":"info","ts":"2026-06-05T09:39:33.836Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"21417b63-e278-47f1-9ac8-03e76275f29a","name":"fail-d7vcr-workers"}
{"level":"info","ts":"2026-06-05T09:39:33.836Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"21417b63-e278-47f1-9ac8-03e76275f29a","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:33.848Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"d6f9833e-ab48-4995-8871-b7c2c23b6da4","certificate":"ray-worker-cert-fail-d7vcr","dnsNames":["fail-d7vcr-worker-svc","localhost","fail-d7vcr-small-group","fail-d7vcr-worker-svc.test-ns-l4cdh.svc","fail-d7vcr-worker-svc.test-ns-l4cdh.svc.cluster.local","*.fail-d7vcr-worker-svc.test-ns-l4cdh.svc","*.fail-d7vcr-worker-svc.test-ns-l4cdh.svc.cluster.local","*-worker-*.test-ns-l4cdh.svc","*-worker-*.test-ns-l4cdh.svc.cluster.local"],"ipAddresses":["10.132.0.16","127.0.0.1"]}
{"level":"info","ts":"2026-06-05T09:39:33.848Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"d6f9833e-ab48-4995-8871-b7c2c23b6da4","rayCluster":"fail-d7vcr","podIPs":["10.132.0.16"]}
{"level":"info","ts":"2026-06-05T09:39:34.159Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"80ad4f08-d794-49d7-8a1b-8177c8db75fa"}
{"level":"info","ts":"2026-06-05T09:39:34.159Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"80ad4f08-d794-49d7-8a1b-8177c8db75fa"}
{"level":"info","ts":"2026-06-05T09:39:34.159Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"80ad4f08-d794-49d7-8a1b-8177c8db75fa","1 head service found":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:34.159Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"80ad4f08-d794-49d7-8a1b-8177c8db75fa","Found 1 head Pod":"fail-d7vcr-head-8br8s","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:39:34.159Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"80ad4f08-d794-49d7-8a1b-8177c8db75fa","head Pod":"fail-d7vcr-head-8br8s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-d7vcr-head-8br8s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:34.159Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"80ad4f08-d794-49d7-8a1b-8177c8db75fa","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:39:34.159Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"80ad4f08-d794-49d7-8a1b-8177c8db75fa","worker Pod":"fail-d7vcr-small-group-worker-crtnw","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-d7vcr-small-group-worker-crtnw. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:34.159Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"80ad4f08-d794-49d7-8a1b-8177c8db75fa","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:39:34.159Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"80ad4f08-d794-49d7-8a1b-8177c8db75fa","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:39:34.159Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"80ad4f08-d794-49d7-8a1b-8177c8db75fa","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:39:34.159Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"80ad4f08-d794-49d7-8a1b-8177c8db75fa","podName":"fail-d7vcr-small-group-worker-crtnw","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:39:34.159Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"80ad4f08-d794-49d7-8a1b-8177c8db75fa","podName":"fail-d7vcr-small-group-worker-crtnw","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:39:34.160Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"80ad4f08-d794-49d7-8a1b-8177c8db75fa","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:39:34.160Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"80ad4f08-d794-49d7-8a1b-8177c8db75fa","seconds":300}
{"level":"info","ts":"2026-06-05T09:39:35.813Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"6045f09d-abd8-41d8-890a-ad57c145aca2"}
{"level":"info","ts":"2026-06-05T09:39:35.813Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"6045f09d-abd8-41d8-890a-ad57c145aca2"}
{"level":"info","ts":"2026-06-05T09:39:35.813Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"6045f09d-abd8-41d8-890a-ad57c145aca2","1 head service found":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:35.813Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"6045f09d-abd8-41d8-890a-ad57c145aca2","Found 1 head Pod":"fail-d7vcr-head-8br8s","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:39:35.813Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"6045f09d-abd8-41d8-890a-ad57c145aca2","head Pod":"fail-d7vcr-head-8br8s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-d7vcr-head-8br8s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:35.813Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"6045f09d-abd8-41d8-890a-ad57c145aca2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:39:35.813Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"6045f09d-abd8-41d8-890a-ad57c145aca2","worker Pod":"fail-d7vcr-small-group-worker-crtnw","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-d7vcr-small-group-worker-crtnw. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:35.813Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"6045f09d-abd8-41d8-890a-ad57c145aca2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:39:35.813Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"6045f09d-abd8-41d8-890a-ad57c145aca2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:39:35.813Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"6045f09d-abd8-41d8-890a-ad57c145aca2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:39:35.813Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"6045f09d-abd8-41d8-890a-ad57c145aca2","podName":"fail-d7vcr-head-8br8s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:39:35.813Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"6045f09d-abd8-41d8-890a-ad57c145aca2","podName":"fail-d7vcr-head-8br8s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:39:35.813Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"6045f09d-abd8-41d8-890a-ad57c145aca2","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:39:35.813Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"6045f09d-abd8-41d8-890a-ad57c145aca2","seconds":300}
{"level":"info","ts":"2026-06-05T09:39:36.007Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"fbd0d157-700d-43c8-9143-0b6a946a4996","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:36.007Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"fbd0d157-700d-43c8-9143-0b6a946a4996","name":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:36.007Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"fbd0d157-700d-43c8-9143-0b6a946a4996","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:36.007Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"fbd0d157-700d-43c8-9143-0b6a946a4996"}
{"level":"info","ts":"2026-06-05T09:39:36.007Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"fbd0d157-700d-43c8-9143-0b6a946a4996","RayCluster":"fail-d7vcr","State":""}
{"level":"info","ts":"2026-06-05T09:39:36.548Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"0487b7a9-71cc-4680-ad4f-a6aa24348674","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:39:36.548Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"0487b7a9-71cc-4680-ad4f-a6aa24348674","pod":"raycluster-head-vjd7d","ip":"10.134.0.20"}
{"level":"info","ts":"2026-06-05T09:39:36.548Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"0487b7a9-71cc-4680-ad4f-a6aa24348674","pod":"raycluster-small-group-worker-j9pbw","ip":"10.133.0.35"}
{"level":"info","ts":"2026-06-05T09:39:36.548Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"0487b7a9-71cc-4680-ad4f-a6aa24348674","rayCluster":"raycluster","podIPs":["10.134.0.20","10.133.0.35"]}
{"level":"info","ts":"2026-06-05T09:39:38.834Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"e0dc0590-2fbb-447f-be93-8c3d8f45473a"}
{"level":"info","ts":"2026-06-05T09:39:38.834Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"e0dc0590-2fbb-447f-be93-8c3d8f45473a"}
{"level":"info","ts":"2026-06-05T09:39:38.834Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"e0dc0590-2fbb-447f-be93-8c3d8f45473a","1 head service found":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:38.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"e0dc0590-2fbb-447f-be93-8c3d8f45473a","Found 1 head Pod":"fail-d7vcr-head-8br8s","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:39:38.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"e0dc0590-2fbb-447f-be93-8c3d8f45473a","head Pod":"fail-d7vcr-head-8br8s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-d7vcr-head-8br8s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:38.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"e0dc0590-2fbb-447f-be93-8c3d8f45473a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:39:38.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"e0dc0590-2fbb-447f-be93-8c3d8f45473a","worker Pod":"fail-d7vcr-small-group-worker-crtnw","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-d7vcr-small-group-worker-crtnw. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:38.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"e0dc0590-2fbb-447f-be93-8c3d8f45473a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:39:38.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"e0dc0590-2fbb-447f-be93-8c3d8f45473a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:39:38.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"e0dc0590-2fbb-447f-be93-8c3d8f45473a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:39:38.834Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"e0dc0590-2fbb-447f-be93-8c3d8f45473a","podName":"fail-d7vcr-head-8br8s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:39:38.834Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"e0dc0590-2fbb-447f-be93-8c3d8f45473a","podName":"fail-d7vcr-head-8br8s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:39:38.834Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"e0dc0590-2fbb-447f-be93-8c3d8f45473a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:39:38.834Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"e0dc0590-2fbb-447f-be93-8c3d8f45473a","seconds":300}
{"level":"info","ts":"2026-06-05T09:39:39.008Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"98d66a3b-4ba3-4bb3-b77f-9aad1dc60465","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:39.008Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"98d66a3b-4ba3-4bb3-b77f-9aad1dc60465","name":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:39.008Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"98d66a3b-4ba3-4bb3-b77f-9aad1dc60465","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:39.008Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"98d66a3b-4ba3-4bb3-b77f-9aad1dc60465"}
{"level":"info","ts":"2026-06-05T09:39:39.008Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"98d66a3b-4ba3-4bb3-b77f-9aad1dc60465","RayCluster":"fail-d7vcr","State":""}
{"level":"info","ts":"2026-06-05T09:39:39.834Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"00518112-618f-483c-b65b-be94abe23403"}
{"level":"info","ts":"2026-06-05T09:39:39.834Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"00518112-618f-483c-b65b-be94abe23403"}
{"level":"info","ts":"2026-06-05T09:39:39.834Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"00518112-618f-483c-b65b-be94abe23403","1 head service found":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:39.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"00518112-618f-483c-b65b-be94abe23403","Found 1 head Pod":"fail-d7vcr-head-8br8s","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:39:39.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"00518112-618f-483c-b65b-be94abe23403","head Pod":"fail-d7vcr-head-8br8s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-d7vcr-head-8br8s. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:39.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"00518112-618f-483c-b65b-be94abe23403","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:39:39.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"00518112-618f-483c-b65b-be94abe23403","worker Pod":"fail-d7vcr-small-group-worker-crtnw","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-d7vcr-small-group-worker-crtnw. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:39.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"00518112-618f-483c-b65b-be94abe23403","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:39:39.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"00518112-618f-483c-b65b-be94abe23403","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:39:39.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"00518112-618f-483c-b65b-be94abe23403","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:39:39.834Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"00518112-618f-483c-b65b-be94abe23403","podName":"fail-d7vcr-head-8br8s","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:39:32Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:39:39.834Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"00518112-618f-483c-b65b-be94abe23403","podName":"fail-d7vcr-head-8br8s","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:39:32Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:39:39.834Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"00518112-618f-483c-b65b-be94abe23403","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-05T09:39:39.834Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"00518112-618f-483c-b65b-be94abe23403","name":"fail-d7vcr","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.16","serviceIP":"10.132.0.16","podName":"fail-d7vcr-head-8br8s","serviceName":"fail-d7vcr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.16","serviceIP":"10.132.0.16","podName":"fail-d7vcr-head-8br8s","serviceName":"fail-d7vcr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:39:39.849Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"d2652c4d-a1a9-4b79-8cfe-568cd8336118","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:39.849Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"baa52dae-e4ff-4d42-93ea-30b96347b27f","rayCluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:39.849Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"b2a3b953-d0d6-4cb0-a3e6-22d78aff99c2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:39.849Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"b2a3b953-d0d6-4cb0-a3e6-22d78aff99c2","name":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:39.849Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"baa52dae-e4ff-4d42-93ea-30b96347b27f","pod":"fail-d7vcr-head-8br8s","ip":"10.132.0.16"}
{"level":"info","ts":"2026-06-05T09:39:39.849Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"b2a3b953-d0d6-4cb0-a3e6-22d78aff99c2","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:39.849Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"baa52dae-e4ff-4d42-93ea-30b96347b27f","pod":"fail-d7vcr-small-group-worker-crtnw","ip":"10.133.0.36"}
{"level":"info","ts":"2026-06-05T09:39:39.849Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"b2a3b953-d0d6-4cb0-a3e6-22d78aff99c2"}
{"level":"info","ts":"2026-06-05T09:39:39.849Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"b2a3b953-d0d6-4cb0-a3e6-22d78aff99c2","RayCluster":"fail-d7vcr","State":""}
{"level":"info","ts":"2026-06-05T09:39:39.864Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"d2652c4d-a1a9-4b79-8cfe-568cd8336118","name":"fail-d7vcr-head"}
{"level":"info","ts":"2026-06-05T09:39:39.864Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"baa52dae-e4ff-4d42-93ea-30b96347b27f","certificate":"ray-head-cert-fail-d7vcr","dnsNames":["fail-d7vcr-head-svc","localhost","fail-d7vcr-head-svc.test-ns-l4cdh.svc","fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local"],"ipAddresses":["10.132.0.16","10.133.0.36","127.0.0.1"]}
{"level":"info","ts":"2026-06-05T09:39:39.879Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"baa52dae-e4ff-4d42-93ea-30b96347b27f","certificate":"ray-worker-cert-fail-d7vcr","dnsNames":["fail-d7vcr-worker-svc","localhost","fail-d7vcr-small-group","fail-d7vcr-worker-svc.test-ns-l4cdh.svc","fail-d7vcr-worker-svc.test-ns-l4cdh.svc.cluster.local","*.fail-d7vcr-worker-svc.test-ns-l4cdh.svc","*.fail-d7vcr-worker-svc.test-ns-l4cdh.svc.cluster.local","*-worker-*.test-ns-l4cdh.svc","*-worker-*.test-ns-l4cdh.svc.cluster.local"],"ipAddresses":["10.132.0.16","10.133.0.36","127.0.0.1"]}
{"level":"info","ts":"2026-06-05T09:39:39.879Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"baa52dae-e4ff-4d42-93ea-30b96347b27f","rayCluster":"fail-d7vcr","podIPs":["10.132.0.16","10.133.0.36"]}
{"level":"info","ts":"2026-06-05T09:39:39.881Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"d2652c4d-a1a9-4b79-8cfe-568cd8336118","name":"fail-d7vcr-workers"}
{"level":"info","ts":"2026-06-05T09:39:39.881Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"d2652c4d-a1a9-4b79-8cfe-568cd8336118","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:41.849Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f0de85df-9225-4318-9d3d-4fcd6b6d20b4"}
{"level":"info","ts":"2026-06-05T09:39:41.849Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f0de85df-9225-4318-9d3d-4fcd6b6d20b4"}
{"level":"info","ts":"2026-06-05T09:39:41.849Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f0de85df-9225-4318-9d3d-4fcd6b6d20b4","1 head service found":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:41.850Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f0de85df-9225-4318-9d3d-4fcd6b6d20b4","Found 1 head Pod":"fail-d7vcr-head-8br8s","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:39:41.850Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f0de85df-9225-4318-9d3d-4fcd6b6d20b4","head Pod":"fail-d7vcr-head-8br8s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-d7vcr-head-8br8s. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:41.850Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f0de85df-9225-4318-9d3d-4fcd6b6d20b4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:39:41.850Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f0de85df-9225-4318-9d3d-4fcd6b6d20b4","worker Pod":"fail-d7vcr-small-group-worker-crtnw","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-d7vcr-small-group-worker-crtnw. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:41.850Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f0de85df-9225-4318-9d3d-4fcd6b6d20b4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:39:41.850Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f0de85df-9225-4318-9d3d-4fcd6b6d20b4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:39:41.850Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f0de85df-9225-4318-9d3d-4fcd6b6d20b4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:39:41.850Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f0de85df-9225-4318-9d3d-4fcd6b6d20b4","podName":"fail-d7vcr-head-8br8s","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:39:32Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:39:41.850Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f0de85df-9225-4318-9d3d-4fcd6b6d20b4","podName":"fail-d7vcr-head-8br8s","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:39:32Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:39:41.850Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f0de85df-9225-4318-9d3d-4fcd6b6d20b4","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:39:41.850Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f0de85df-9225-4318-9d3d-4fcd6b6d20b4","seconds":300}
{"level":"info","ts":"2026-06-05T09:39:42.009Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"0e4398e9-db97-4068-98d7-410c27a60d2c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:42.009Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"0e4398e9-db97-4068-98d7-410c27a60d2c","name":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:42.009Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"0e4398e9-db97-4068-98d7-410c27a60d2c","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:42.009Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"0e4398e9-db97-4068-98d7-410c27a60d2c"}
{"level":"info","ts":"2026-06-05T09:39:42.009Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"0e4398e9-db97-4068-98d7-410c27a60d2c","RayCluster":"fail-d7vcr","State":""}
{"level":"info","ts":"2026-06-05T09:39:43.202Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"003e0c5d-12ce-435d-8aef-3e39302e816f"}
{"level":"info","ts":"2026-06-05T09:39:43.202Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"003e0c5d-12ce-435d-8aef-3e39302e816f"}
{"level":"info","ts":"2026-06-05T09:39:43.202Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"003e0c5d-12ce-435d-8aef-3e39302e816f","1 head service found":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:43.202Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"003e0c5d-12ce-435d-8aef-3e39302e816f","Found 1 head Pod":"fail-d7vcr-head-8br8s","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:39:43.202Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"003e0c5d-12ce-435d-8aef-3e39302e816f","head Pod":"fail-d7vcr-head-8br8s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-d7vcr-head-8br8s. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:43.202Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"003e0c5d-12ce-435d-8aef-3e39302e816f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:39:43.202Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"003e0c5d-12ce-435d-8aef-3e39302e816f","worker Pod":"fail-d7vcr-small-group-worker-crtnw","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-d7vcr-small-group-worker-crtnw. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:43.202Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"003e0c5d-12ce-435d-8aef-3e39302e816f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:39:43.202Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"003e0c5d-12ce-435d-8aef-3e39302e816f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:39:43.202Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"003e0c5d-12ce-435d-8aef-3e39302e816f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:39:43.202Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"003e0c5d-12ce-435d-8aef-3e39302e816f","podName":"fail-d7vcr-head-8br8s","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:39:32Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:39:43.202Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"003e0c5d-12ce-435d-8aef-3e39302e816f","podName":"fail-d7vcr-head-8br8s","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:39:32Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:39:43.202Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"003e0c5d-12ce-435d-8aef-3e39302e816f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:39:43.202Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"003e0c5d-12ce-435d-8aef-3e39302e816f","seconds":300}
{"level":"info","ts":"2026-06-05T09:39:44.208Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"9ee62a6e-846b-43e3-9416-c07ba1be7a68"}
{"level":"info","ts":"2026-06-05T09:39:44.208Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"9ee62a6e-846b-43e3-9416-c07ba1be7a68"}
{"level":"info","ts":"2026-06-05T09:39:44.208Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"9ee62a6e-846b-43e3-9416-c07ba1be7a68","1 head service found":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:44.208Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"9ee62a6e-846b-43e3-9416-c07ba1be7a68","Found 1 head Pod":"fail-d7vcr-head-8br8s","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:39:44.208Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"9ee62a6e-846b-43e3-9416-c07ba1be7a68","head Pod":"fail-d7vcr-head-8br8s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-d7vcr-head-8br8s. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:44.208Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"9ee62a6e-846b-43e3-9416-c07ba1be7a68","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:39:44.208Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"9ee62a6e-846b-43e3-9416-c07ba1be7a68","worker Pod":"fail-d7vcr-small-group-worker-crtnw","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-d7vcr-small-group-worker-crtnw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:44.208Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"9ee62a6e-846b-43e3-9416-c07ba1be7a68","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:39:44.208Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"9ee62a6e-846b-43e3-9416-c07ba1be7a68","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:39:44.208Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"9ee62a6e-846b-43e3-9416-c07ba1be7a68","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:39:44.208Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"9ee62a6e-846b-43e3-9416-c07ba1be7a68","podName":"fail-d7vcr-head-8br8s","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:39:32Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:39:44.208Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"9ee62a6e-846b-43e3-9416-c07ba1be7a68","podName":"fail-d7vcr-head-8br8s","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:39:32Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:39:44.208Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"9ee62a6e-846b-43e3-9416-c07ba1be7a68","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-05T09:39:44.208Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"9ee62a6e-846b-43e3-9416-c07ba1be7a68","name":"fail-d7vcr","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.16","serviceIP":"10.132.0.16","podName":"fail-d7vcr-head-8br8s","serviceName":"fail-d7vcr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.16","serviceIP":"10.132.0.16","podName":"fail-d7vcr-head-8br8s","serviceName":"fail-d7vcr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:39:44.223Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"69fb4882-a29f-4d5d-bc82-2d577d96f8f0","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:44.223Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"d7e1de12-55c7-4445-a00d-2b023f330419","rayCluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:44.223Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"11c58754-674b-44e2-91be-7663edcb83f9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:44.223Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"11c58754-674b-44e2-91be-7663edcb83f9","name":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:44.223Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"11c58754-674b-44e2-91be-7663edcb83f9","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:44.223Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"d7e1de12-55c7-4445-a00d-2b023f330419","pod":"fail-d7vcr-head-8br8s","ip":"10.132.0.16"}
{"level":"info","ts":"2026-06-05T09:39:44.223Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"d7e1de12-55c7-4445-a00d-2b023f330419","pod":"fail-d7vcr-small-group-worker-crtnw","ip":"10.133.0.36"}
{"level":"info","ts":"2026-06-05T09:39:44.223Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"d7e1de12-55c7-4445-a00d-2b023f330419","rayCluster":"fail-d7vcr","podIPs":["10.132.0.16","10.133.0.36"]}
{"level":"info","ts":"2026-06-05T09:39:44.223Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"11c58754-674b-44e2-91be-7663edcb83f9"}
{"level":"info","ts":"2026-06-05T09:39:44.223Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"11c58754-674b-44e2-91be-7663edcb83f9","RayCluster":"fail-d7vcr","State":""}
{"level":"info","ts":"2026-06-05T09:39:44.239Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"69fb4882-a29f-4d5d-bc82-2d577d96f8f0","name":"fail-d7vcr-head"}
{"level":"info","ts":"2026-06-05T09:39:44.249Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"69fb4882-a29f-4d5d-bc82-2d577d96f8f0","name":"fail-d7vcr-workers"}
{"level":"info","ts":"2026-06-05T09:39:44.250Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"69fb4882-a29f-4d5d-bc82-2d577d96f8f0","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:45.010Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"59820ec3-04ed-495d-bcf3-9f67137f7d8b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:45.010Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"59820ec3-04ed-495d-bcf3-9f67137f7d8b","name":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:45.010Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"59820ec3-04ed-495d-bcf3-9f67137f7d8b","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:45.010Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"59820ec3-04ed-495d-bcf3-9f67137f7d8b"}
{"level":"info","ts":"2026-06-05T09:39:45.010Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"59820ec3-04ed-495d-bcf3-9f67137f7d8b","RayCluster":"fail-d7vcr","State":""}
{"level":"info","ts":"2026-06-05T09:39:46.224Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"7c2f5c71-fd3f-48fd-a784-701335ef8e7e"}
{"level":"info","ts":"2026-06-05T09:39:46.224Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"7c2f5c71-fd3f-48fd-a784-701335ef8e7e"}
{"level":"info","ts":"2026-06-05T09:39:46.224Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"7c2f5c71-fd3f-48fd-a784-701335ef8e7e","1 head service found":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:46.224Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"7c2f5c71-fd3f-48fd-a784-701335ef8e7e","Found 1 head Pod":"fail-d7vcr-head-8br8s","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:39:46.224Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"7c2f5c71-fd3f-48fd-a784-701335ef8e7e","head Pod":"fail-d7vcr-head-8br8s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-d7vcr-head-8br8s. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:46.224Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"7c2f5c71-fd3f-48fd-a784-701335ef8e7e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:39:46.224Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"7c2f5c71-fd3f-48fd-a784-701335ef8e7e","worker Pod":"fail-d7vcr-small-group-worker-crtnw","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-d7vcr-small-group-worker-crtnw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:46.224Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"7c2f5c71-fd3f-48fd-a784-701335ef8e7e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:39:46.224Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"7c2f5c71-fd3f-48fd-a784-701335ef8e7e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:39:46.224Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"7c2f5c71-fd3f-48fd-a784-701335ef8e7e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:39:46.224Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"7c2f5c71-fd3f-48fd-a784-701335ef8e7e","podName":"fail-d7vcr-head-8br8s","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:39:32Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:39:46.224Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"7c2f5c71-fd3f-48fd-a784-701335ef8e7e","podName":"fail-d7vcr-head-8br8s","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:39:32Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:39:46.224Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"7c2f5c71-fd3f-48fd-a784-701335ef8e7e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:39:46.224Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"7c2f5c71-fd3f-48fd-a784-701335ef8e7e","seconds":300}
{"level":"info","ts":"2026-06-05T09:39:48.012Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"e3cf0a77-3889-4bc3-bd31-31d904a32308","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:48.012Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"e3cf0a77-3889-4bc3-bd31-31d904a32308","name":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:48.012Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"e3cf0a77-3889-4bc3-bd31-31d904a32308","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:48.012Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"e3cf0a77-3889-4bc3-bd31-31d904a32308"}
{"level":"info","ts":"2026-06-05T09:39:48.012Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"e3cf0a77-3889-4bc3-bd31-31d904a32308","RayCluster":"fail-d7vcr","State":""}
{"level":"info","ts":"2026-06-05T09:39:50.858Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"1ea9f302-938e-4a67-8e6c-19074d8b861a"}
{"level":"info","ts":"2026-06-05T09:39:50.858Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"1ea9f302-938e-4a67-8e6c-19074d8b861a"}
{"level":"info","ts":"2026-06-05T09:39:50.858Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"1ea9f302-938e-4a67-8e6c-19074d8b861a","1 head service found":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:50.858Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"1ea9f302-938e-4a67-8e6c-19074d8b861a","Found 1 head Pod":"fail-d7vcr-head-8br8s","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:39:50.858Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"1ea9f302-938e-4a67-8e6c-19074d8b861a","head Pod":"fail-d7vcr-head-8br8s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-d7vcr-head-8br8s. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:50.858Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"1ea9f302-938e-4a67-8e6c-19074d8b861a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:39:50.858Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"1ea9f302-938e-4a67-8e6c-19074d8b861a","worker Pod":"fail-d7vcr-small-group-worker-crtnw","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-d7vcr-small-group-worker-crtnw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:50.858Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"1ea9f302-938e-4a67-8e6c-19074d8b861a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:39:50.858Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"1ea9f302-938e-4a67-8e6c-19074d8b861a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:39:50.858Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"1ea9f302-938e-4a67-8e6c-19074d8b861a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:39:50.858Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"1ea9f302-938e-4a67-8e6c-19074d8b861a","podName":"fail-d7vcr-small-group-worker-crtnw","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:39:32Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-05T09:39:50.858Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"1ea9f302-938e-4a67-8e6c-19074d8b861a","podName":"fail-d7vcr-small-group-worker-crtnw","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:39:32Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-05T09:39:50.858Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"1ea9f302-938e-4a67-8e6c-19074d8b861a","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:39:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-05T09:39:50.858Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"1ea9f302-938e-4a67-8e6c-19074d8b861a","name":"fail-d7vcr","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.16","serviceIP":"10.132.0.16","podName":"fail-d7vcr-head-8br8s","serviceName":"fail-d7vcr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.16","serviceIP":"10.132.0.16","podName":"fail-d7vcr-head-8br8s","serviceName":"fail-d7vcr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:39:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:39:50.870Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"a2a14fc3-5c9a-48e8-8e6a-31b648262049","rayCluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:50.870Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"29cefb74-7196-4828-a5a0-bb303e94613d","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:50.870Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"58371f95-5f4e-47f5-866d-f258b22e4db3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:50.870Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"58371f95-5f4e-47f5-866d-f258b22e4db3","name":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:50.870Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"a2a14fc3-5c9a-48e8-8e6a-31b648262049","pod":"fail-d7vcr-head-8br8s","ip":"10.132.0.16"}
{"level":"info","ts":"2026-06-05T09:39:50.870Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"a2a14fc3-5c9a-48e8-8e6a-31b648262049","pod":"fail-d7vcr-small-group-worker-crtnw","ip":"10.133.0.36"}
{"level":"info","ts":"2026-06-05T09:39:50.870Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"58371f95-5f4e-47f5-866d-f258b22e4db3","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:50.870Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"a2a14fc3-5c9a-48e8-8e6a-31b648262049","rayCluster":"fail-d7vcr","podIPs":["10.132.0.16","10.133.0.36"]}
{"level":"info","ts":"2026-06-05T09:39:50.870Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"58371f95-5f4e-47f5-866d-f258b22e4db3"}
{"level":"info","ts":"2026-06-05T09:39:50.870Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"58371f95-5f4e-47f5-866d-f258b22e4db3","RayCluster":"fail-d7vcr","State":""}
{"level":"info","ts":"2026-06-05T09:39:50.880Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"29cefb74-7196-4828-a5a0-bb303e94613d","name":"fail-d7vcr-head"}
{"level":"info","ts":"2026-06-05T09:39:50.892Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"29cefb74-7196-4828-a5a0-bb303e94613d","name":"fail-d7vcr-workers"}
{"level":"info","ts":"2026-06-05T09:39:50.892Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"29cefb74-7196-4828-a5a0-bb303e94613d","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:51.012Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"83ece04a-5dee-4542-9fab-c7ad99efaafd","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:51.012Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"83ece04a-5dee-4542-9fab-c7ad99efaafd","name":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:51.012Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"83ece04a-5dee-4542-9fab-c7ad99efaafd","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:51.012Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"83ece04a-5dee-4542-9fab-c7ad99efaafd"}
{"level":"info","ts":"2026-06-05T09:39:51.013Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"83ece04a-5dee-4542-9fab-c7ad99efaafd","RayCluster":"fail-d7vcr","State":""}
{"level":"info","ts":"2026-06-05T09:39:52.871Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"9ef7c452-8df9-4ea8-8f8a-6fb65855d0cc"}
{"level":"info","ts":"2026-06-05T09:39:52.871Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"9ef7c452-8df9-4ea8-8f8a-6fb65855d0cc"}
{"level":"info","ts":"2026-06-05T09:39:52.871Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"9ef7c452-8df9-4ea8-8f8a-6fb65855d0cc","1 head service found":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:52.871Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"9ef7c452-8df9-4ea8-8f8a-6fb65855d0cc","Found 1 head Pod":"fail-d7vcr-head-8br8s","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:39:52.871Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"9ef7c452-8df9-4ea8-8f8a-6fb65855d0cc","head Pod":"fail-d7vcr-head-8br8s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-d7vcr-head-8br8s. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:52.871Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"9ef7c452-8df9-4ea8-8f8a-6fb65855d0cc","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:39:52.871Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"9ef7c452-8df9-4ea8-8f8a-6fb65855d0cc","worker Pod":"fail-d7vcr-small-group-worker-crtnw","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-d7vcr-small-group-worker-crtnw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:52.871Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"9ef7c452-8df9-4ea8-8f8a-6fb65855d0cc","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:39:52.871Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"9ef7c452-8df9-4ea8-8f8a-6fb65855d0cc","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:39:52.871Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"9ef7c452-8df9-4ea8-8f8a-6fb65855d0cc","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:39:52.871Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"9ef7c452-8df9-4ea8-8f8a-6fb65855d0cc","podName":"fail-d7vcr-small-group-worker-crtnw","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:39:32Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-05T09:39:52.871Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"9ef7c452-8df9-4ea8-8f8a-6fb65855d0cc","podName":"fail-d7vcr-small-group-worker-crtnw","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:39:32Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-05T09:39:52.871Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"9ef7c452-8df9-4ea8-8f8a-6fb65855d0cc","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:39:52.871Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"9ef7c452-8df9-4ea8-8f8a-6fb65855d0cc","seconds":300}
{"level":"info","ts":"2026-06-05T09:39:54.013Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"d6bc042a-8864-4fb2-a3fc-cc5fdc597e2e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:54.013Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"d6bc042a-8864-4fb2-a3fc-cc5fdc597e2e","name":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:54.013Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"d6bc042a-8864-4fb2-a3fc-cc5fdc597e2e","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:54.013Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"d6bc042a-8864-4fb2-a3fc-cc5fdc597e2e"}
{"level":"info","ts":"2026-06-05T09:39:54.013Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"d6bc042a-8864-4fb2-a3fc-cc5fdc597e2e","RayCluster":"fail-d7vcr","State":""}
{"level":"info","ts":"2026-06-05T09:39:55.235Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"03f4e015-2f71-4222-b27f-102f56fbf0d0"}
{"level":"info","ts":"2026-06-05T09:39:55.235Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"03f4e015-2f71-4222-b27f-102f56fbf0d0"}
{"level":"info","ts":"2026-06-05T09:39:55.235Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"03f4e015-2f71-4222-b27f-102f56fbf0d0","1 head service found":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:55.235Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"03f4e015-2f71-4222-b27f-102f56fbf0d0","Found 1 head Pod":"fail-d7vcr-head-8br8s","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:39:55.235Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"03f4e015-2f71-4222-b27f-102f56fbf0d0","head Pod":"fail-d7vcr-head-8br8s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-d7vcr-head-8br8s. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:55.235Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"03f4e015-2f71-4222-b27f-102f56fbf0d0","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:39:55.235Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"03f4e015-2f71-4222-b27f-102f56fbf0d0","worker Pod":"fail-d7vcr-small-group-worker-crtnw","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-d7vcr-small-group-worker-crtnw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:55.235Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"03f4e015-2f71-4222-b27f-102f56fbf0d0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:39:55.235Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"03f4e015-2f71-4222-b27f-102f56fbf0d0","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:39:55.235Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"03f4e015-2f71-4222-b27f-102f56fbf0d0","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:39:55.235Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"03f4e015-2f71-4222-b27f-102f56fbf0d0","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-06-05T09:39:55.235Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"03f4e015-2f71-4222-b27f-102f56fbf0d0","name":"fail-d7vcr","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.16","serviceIP":"10.132.0.16","podName":"fail-d7vcr-head-8br8s","serviceName":"fail-d7vcr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:39:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:55Z","stateTransitionTimes":{"ready":"2026-06-05T09:39:55Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.16","serviceIP":"10.132.0.16","podName":"fail-d7vcr-head-8br8s","serviceName":"fail-d7vcr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:39:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:39:55Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:39:55.248Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"6363ad8e-c39d-431d-b185-308a781cdc4a","rayCluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:55.248Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"c4c36ef8-4978-4c92-971d-d6a232bdc81f","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:55.248Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"b5460d00-ccf6-4a84-83d6-f4222b8b6a6b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:55.248Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"b5460d00-ccf6-4a84-83d6-f4222b8b6a6b","name":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:55.248Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"6363ad8e-c39d-431d-b185-308a781cdc4a","pod":"fail-d7vcr-head-8br8s","ip":"10.132.0.16"}
{"level":"info","ts":"2026-06-05T09:39:55.248Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"b5460d00-ccf6-4a84-83d6-f4222b8b6a6b","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:55.248Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"6363ad8e-c39d-431d-b185-308a781cdc4a","pod":"fail-d7vcr-small-group-worker-crtnw","ip":"10.133.0.36"}
{"level":"info","ts":"2026-06-05T09:39:55.249Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"6363ad8e-c39d-431d-b185-308a781cdc4a","rayCluster":"fail-d7vcr","podIPs":["10.132.0.16","10.133.0.36"]}
{"level":"info","ts":"2026-06-05T09:39:55.249Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"b5460d00-ccf6-4a84-83d6-f4222b8b6a6b"}
{"level":"info","ts":"2026-06-05T09:39:55.249Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"b5460d00-ccf6-4a84-83d6-f4222b8b6a6b","head service name":"fail-d7vcr-head-svc","namespace":"test-ns-l4cdh"}
{"level":"info","ts":"2026-06-05T09:39:55.249Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"b5460d00-ccf6-4a84-83d6-f4222b8b6a6b","head service URL":"fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-05T09:39:55.249Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"b5460d00-ccf6-4a84-83d6-f4222b8b6a6b","SubmissionMode":"HTTPMode","RayCluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:55.249Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"b5460d00-ccf6-4a84-83d6-f4222b8b6a6b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-t7t9k","rayClusterName":"fail-d7vcr","jobDeploymentStatus":"Initializing","startTime":"2026-06-05T09:39:29Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-t7t9k","rayClusterName":"fail-d7vcr","dashboardURL":"fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-05T09:39:29Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:39:55.249Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"b5460d00-ccf6-4a84-83d6-f4222b8b6a6b","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-05T09:39:55.260Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"c7db67c2-695f-47a5-b89e-2e6672a1afc1","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:55.260Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"c7db67c2-695f-47a5-b89e-2e6672a1afc1","name":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:55.260Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"c7db67c2-695f-47a5-b89e-2e6672a1afc1","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:55.261Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"c7db67c2-695f-47a5-b89e-2e6672a1afc1"}
{"level":"info","ts":"2026-06-05T09:39:55.264Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"c4c36ef8-4978-4c92-971d-d6a232bdc81f","name":"fail-d7vcr-head"}
{"level":"info","ts":"2026-06-05T09:39:55.269Z","logger":"controllers.RayJob","msg":"The Ray job was not found. Submit a Ray job via an HTTP request.","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"c7db67c2-695f-47a5-b89e-2e6672a1afc1","JobId":"fail-t7t9k"}
{"level":"info","ts":"2026-06-05T09:39:55.269Z","logger":"controllers.RayJob","msg":"Submit a ray job","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"c7db67c2-695f-47a5-b89e-2e6672a1afc1","rayJob":"fail","jobInfo":"{\"entrypoint\":\"python /home/ray/jobs/fail.py\",\"submission_id\":\"fail-t7t9k\"}"}
{"level":"info","ts":"2026-06-05T09:39:55.277Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"c4c36ef8-4978-4c92-971d-d6a232bdc81f","name":"fail-d7vcr-workers"}
{"level":"info","ts":"2026-06-05T09:39:55.277Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"c4c36ef8-4978-4c92-971d-d6a232bdc81f","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:57.014Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"38e3c28d-4a4a-4db7-992e-aca57d5461cd","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:57.015Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"38e3c28d-4a4a-4db7-992e-aca57d5461cd","name":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:57.015Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"38e3c28d-4a4a-4db7-992e-aca57d5461cd","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:57.015Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"38e3c28d-4a4a-4db7-992e-aca57d5461cd"}
{"level":"info","ts":"2026-06-05T09:39:57.018Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"38e3c28d-4a4a-4db7-992e-aca57d5461cd","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-t7t9k","rayClusterName":"fail-d7vcr","dashboardURL":"fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-05T09:39:29Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:39:55Z"},"jobId":"fail-t7t9k","rayClusterName":"fail-d7vcr","dashboardURL":"fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-05T09:39:29Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:55Z","stateTransitionTimes":{"ready":"2026-06-05T09:39:55Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.16","serviceIP":"10.132.0.16","podName":"fail-d7vcr-head-8br8s","serviceName":"fail-d7vcr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:39:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:39:55Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:39:57.018Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"38e3c28d-4a4a-4db7-992e-aca57d5461cd","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-05T09:39:57.033Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"c8b7b802-f4de-43ae-9cc6-cf634d08ab8f","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:39:57.033Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"c8b7b802-f4de-43ae-9cc6-cf634d08ab8f","name":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:39:57.033Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"c8b7b802-f4de-43ae-9cc6-cf634d08ab8f","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:39:57.033Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"c8b7b802-f4de-43ae-9cc6-cf634d08ab8f"}
{"level":"info","ts":"2026-06-05T09:39:57.035Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"c8b7b802-f4de-43ae-9cc6-cf634d08ab8f","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:39:55Z"},"jobId":"fail-t7t9k","rayClusterName":"fail-d7vcr","dashboardURL":"fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-05T09:39:29Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:55Z","stateTransitionTimes":{"ready":"2026-06-05T09:39:55Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.16","serviceIP":"10.132.0.16","podName":"fail-d7vcr-head-8br8s","serviceName":"fail-d7vcr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:39:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:39:55Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:39:55Z"},"jobId":"fail-t7t9k","rayClusterName":"fail-d7vcr","dashboardURL":"fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-05T09:39:29Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:55Z","stateTransitionTimes":{"ready":"2026-06-05T09:39:55Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.16","serviceIP":"10.132.0.16","podName":"fail-d7vcr-head-8br8s","serviceName":"fail-d7vcr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:39:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:39:55Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:39:57.249Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f3e8f0ab-fd27-4aab-9dd6-ff70b5d2ba15"}
{"level":"info","ts":"2026-06-05T09:39:57.249Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f3e8f0ab-fd27-4aab-9dd6-ff70b5d2ba15"}
{"level":"info","ts":"2026-06-05T09:39:57.249Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f3e8f0ab-fd27-4aab-9dd6-ff70b5d2ba15","1 head service found":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:39:57.249Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f3e8f0ab-fd27-4aab-9dd6-ff70b5d2ba15","Found 1 head Pod":"fail-d7vcr-head-8br8s","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:39:57.249Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f3e8f0ab-fd27-4aab-9dd6-ff70b5d2ba15","head Pod":"fail-d7vcr-head-8br8s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-d7vcr-head-8br8s. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:57.249Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f3e8f0ab-fd27-4aab-9dd6-ff70b5d2ba15","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:39:57.249Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f3e8f0ab-fd27-4aab-9dd6-ff70b5d2ba15","worker Pod":"fail-d7vcr-small-group-worker-crtnw","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-d7vcr-small-group-worker-crtnw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:39:57.249Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f3e8f0ab-fd27-4aab-9dd6-ff70b5d2ba15","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:39:57.249Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f3e8f0ab-fd27-4aab-9dd6-ff70b5d2ba15","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:39:57.249Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f3e8f0ab-fd27-4aab-9dd6-ff70b5d2ba15","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:39:57.249Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f3e8f0ab-fd27-4aab-9dd6-ff70b5d2ba15","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:39:57.249Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f3e8f0ab-fd27-4aab-9dd6-ff70b5d2ba15","seconds":300}
{"level":"info","ts":"2026-06-05T09:40:00.032Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"97049de8-9e96-468c-89f7-fb8ed2e66dee","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:40:00.032Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"97049de8-9e96-468c-89f7-fb8ed2e66dee","name":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:40:00.032Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"97049de8-9e96-468c-89f7-fb8ed2e66dee","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:40:00.032Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"97049de8-9e96-468c-89f7-fb8ed2e66dee"}
{"level":"info","ts":"2026-06-05T09:40:00.035Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"97049de8-9e96-468c-89f7-fb8ed2e66dee","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:39:55Z"},"jobId":"fail-t7t9k","rayClusterName":"fail-d7vcr","dashboardURL":"fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-05T09:39:29Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:55Z","stateTransitionTimes":{"ready":"2026-06-05T09:39:55Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.16","serviceIP":"10.132.0.16","podName":"fail-d7vcr-head-8br8s","serviceName":"fail-d7vcr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:39:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:39:55Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:39:55Z","endTime":"2026-06-05T09:39:57Z"},"jobId":"fail-t7t9k","rayClusterName":"fail-d7vcr","dashboardURL":"fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Failed","reason":"AppFailed","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-05 09:39:55,837\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-t7t9k: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-05T09:39:29Z","succeeded":0,"failed":1,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:55Z","stateTransitionTimes":{"ready":"2026-06-05T09:39:55Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.16","serviceIP":"10.132.0.16","podName":"fail-d7vcr-head-8br8s","serviceName":"fail-d7vcr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:39:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:39:55Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:40:00.035Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"97049de8-9e96-468c-89f7-fb8ed2e66dee","old JobStatus":"PENDING","new JobStatus":"FAILED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Failed"}
{"level":"info","ts":"2026-06-05T09:40:00.050Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"9aec2553-c79f-4d77-ae7b-108dcfaaa5c2","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:40:00.050Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"9aec2553-c79f-4d77-ae7b-108dcfaaa5c2","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-05 09:40:00 +0000 UTC","Now":"2026-06-05T09:40:00.050Z","ShutdownTime":"2026-06-05T09:40:00.000Z"}
{"level":"info","ts":"2026-06-05T09:40:00.122Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"9281be8b-c894-4442-b843-4cb518e2cc12","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:40:00.122Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"9281be8b-c894-4442-b843-4cb518e2cc12","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-05T09:40:00.130Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"9281be8b-c894-4442-b843-4cb518e2cc12"}
{"level":"info","ts":"2026-06-05T09:40:00.130Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"9281be8b-c894-4442-b843-4cb518e2cc12","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-05T09:40:00.130Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"9281be8b-c894-4442-b843-4cb518e2cc12","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-m4mp7","rayClusterName":"stop-mmlfk","jobDeploymentStatus":"Initializing","startTime":"2026-06-05T09:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:40:00.130Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"9281be8b-c894-4442-b843-4cb518e2cc12","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-05T09:40:00.142Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"e8ab03a8-de00-4a50-b692-aaa24ab53196","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:40:00.142Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"e8ab03a8-de00-4a50-b692-aaa24ab53196"}
{"level":"info","ts":"2026-06-05T09:40:00.142Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"e8ab03a8-de00-4a50-b692-aaa24ab53196","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-05T09:40:00.142Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"e8ab03a8-de00-4a50-b692-aaa24ab53196","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ppwcw","rayClusterName":"stop-gnlrs","jobDeploymentStatus":"Initializing","startTime":"2026-06-05T09:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:40:00.142Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"e8ab03a8-de00-4a50-b692-aaa24ab53196","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-05T09:40:00.149Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"e8ab03a8-de00-4a50-b692-aaa24ab53196","error":"Operation cannot be fulfilled on rayjobs.ray.io \"stop\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-06-05T09:40:00.149Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"e8ab03a8-de00-4a50-b692-aaa24ab53196"}
{"level":"error","ts":"2026-06-05T09:40:00.149Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"e8ab03a8-de00-4a50-b692-aaa24ab53196","error":"Operation cannot be fulfilled on rayjobs.ray.io \"stop\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:40:00.149Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"5139e66d-7251-41fc-936f-5c6598e45621","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:40:00.149Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"5139e66d-7251-41fc-936f-5c6598e45621","name":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:00.149Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"5139e66d-7251-41fc-936f-5c6598e45621","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:40:00.149Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"5139e66d-7251-41fc-936f-5c6598e45621","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:40:00.154Z","logger":"raycluster-resource","msg":"default","name":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:00.154Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"stop-mmlfk","namespace":"test-ns-l4cdh"}
{"level":"info","ts":"2026-06-05T09:40:00.162Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"5139e66d-7251-41fc-936f-5c6598e45621","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:40:00.162Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"cd0d3285-dfb4-4cec-b053-850408068de6","rayCluster":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:00.162Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"cd0d3285-dfb4-4cec-b053-850408068de6","rayCluster":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:00.162Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"a19a95e1-8ae2-478c-af18-43db47ec1853","namespacedName":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:40:00.162Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"5139e66d-7251-41fc-936f-5c6598e45621"}
{"level":"info","ts":"2026-06-05T09:40:00.162Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"5139e66d-7251-41fc-936f-5c6598e45621","RayCluster":"stop-mmlfk","State":""}
{"level":"info","ts":"2026-06-05T09:40:00.162Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"a19a95e1-8ae2-478c-af18-43db47ec1853","mode":"IntegratedOAuth","cluster":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:00.162Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"a19a95e1-8ae2-478c-af18-43db47ec1853","cluster":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:00.162Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"837cc4f2-754a-4684-8b09-e594df203a31"}
{"level":"info","ts":"2026-06-05T09:40:00.162Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"6d4ddef1-b91a-440d-97bc-ec2d7db3f856","cluster":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:00.162Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"a19a95e1-8ae2-478c-af18-43db47ec1853","cluster":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:00.162Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"837cc4f2-754a-4684-8b09-e594df203a31"}
{"level":"info","ts":"2026-06-05T09:40:00.162Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"7ac32c1e-f60c-4aa8-8d45-95fec2ebcab1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:40:00.162Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"7ac32c1e-f60c-4aa8-8d45-95fec2ebcab1","name":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:00.163Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"7ac32c1e-f60c-4aa8-8d45-95fec2ebcab1","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:40:00.163Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"7ac32c1e-f60c-4aa8-8d45-95fec2ebcab1"}
{"level":"info","ts":"2026-06-05T09:40:00.163Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"7ac32c1e-f60c-4aa8-8d45-95fec2ebcab1","RayCluster":"stop-mmlfk","State":""}
{"level":"info","ts":"2026-06-05T09:40:00.206Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"837cc4f2-754a-4684-8b09-e594df203a31","name":"stop-mmlfk-head-svc"}
{"level":"info","ts":"2026-06-05T09:40:00.206Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"837cc4f2-754a-4684-8b09-e594df203a31","error":"head certificate secret ray-head-secret-stop-mmlfk not found"}
{"level":"info","ts":"2026-06-05T09:40:00.206Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"cd0d3285-dfb4-4cec-b053-850408068de6","rayCluster":"stop-mmlfk"}
{"level":"error","ts":"2026-06-05T09:40:00.206Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"837cc4f2-754a-4684-8b09-e594df203a31","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-mmlfk not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:40:00.206Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"cd0d3285-dfb4-4cec-b053-850408068de6","rayCluster":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:00.206Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"837cc4f2-754a-4684-8b09-e594df203a31","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-mmlfk not found"}
{"level":"info","ts":"2026-06-05T09:40:00.206Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"837cc4f2-754a-4684-8b09-e594df203a31","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"stop-mmlfk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:40:00.206Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"837cc4f2-754a-4684-8b09-e594df203a31","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"stop-mmlfk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:40:00.206Z","logger":"raycluster-resource","msg":"default","name":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:00.206Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"837cc4f2-754a-4684-8b09-e594df203a31","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"stop-mmlfk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:40:00.206Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"837cc4f2-754a-4684-8b09-e594df203a31","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-05T09:40:00.206Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"stop-mmlfk","namespace":"test-ns-l4cdh"}
{"level":"info","ts":"2026-06-05T09:40:00.206Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"837cc4f2-754a-4684-8b09-e594df203a31","name":"stop-mmlfk","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:40:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-mmlfk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:40:00.214Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"6d4ddef1-b91a-440d-97bc-ec2d7db3f856","cluster":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:00.214Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"e3e0ce89-8a05-40d0-b50b-79b2b466bbfb","cluster":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:00.217Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"cd0d3285-dfb4-4cec-b053-850408068de6"}
{"level":"info","ts":"2026-06-05T09:40:00.218Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"cd0d3285-dfb4-4cec-b053-850408068de6","rayCluster":"stop-mmlfk","secret":"ca-secret"}
{"level":"info","ts":"2026-06-05T09:40:00.218Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"cd0d3285-dfb4-4cec-b053-850408068de6","rayCluster":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:00.218Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"01e2bf74-bf25-409f-ac1d-613eda410eb6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:40:00.218Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"01e2bf74-bf25-409f-ac1d-613eda410eb6","name":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:00.218Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"01e2bf74-bf25-409f-ac1d-613eda410eb6","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:40:00.218Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"01e2bf74-bf25-409f-ac1d-613eda410eb6"}
{"level":"info","ts":"2026-06-05T09:40:00.218Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"01e2bf74-bf25-409f-ac1d-613eda410eb6","RayCluster":"stop-mmlfk","State":""}
{"level":"info","ts":"2026-06-05T09:40:00.218Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"837cc4f2-754a-4684-8b09-e594df203a31","name":"stop-mmlfk","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-mmlfk\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-l4cdh","name":"stop-mmlfk"}}
{"level":"info","ts":"2026-06-05T09:40:00.218Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"837cc4f2-754a-4684-8b09-e594df203a31"}
{"level":"error","ts":"2026-06-05T09:40:00.218Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"837cc4f2-754a-4684-8b09-e594df203a31","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-mmlfk not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:40:00.219Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"69144ad3-7723-478e-966d-8e0a2bbd994c"}
{"level":"info","ts":"2026-06-05T09:40:00.219Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"69144ad3-7723-478e-966d-8e0a2bbd994c"}
{"level":"info","ts":"2026-06-05T09:40:00.219Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"69144ad3-7723-478e-966d-8e0a2bbd994c","1 head service found":"stop-mmlfk-head-svc"}
{"level":"info","ts":"2026-06-05T09:40:00.219Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"69144ad3-7723-478e-966d-8e0a2bbd994c","error":"head certificate secret ray-head-secret-stop-mmlfk not found"}
{"level":"error","ts":"2026-06-05T09:40:00.219Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"69144ad3-7723-478e-966d-8e0a2bbd994c","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-mmlfk not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:40:00.219Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"69144ad3-7723-478e-966d-8e0a2bbd994c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-mmlfk not found"}
{"level":"info","ts":"2026-06-05T09:40:00.219Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"69144ad3-7723-478e-966d-8e0a2bbd994c","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"stop-mmlfk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:40:00.219Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"69144ad3-7723-478e-966d-8e0a2bbd994c","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"stop-mmlfk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:40:00.219Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"69144ad3-7723-478e-966d-8e0a2bbd994c","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"stop-mmlfk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:40:00.219Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"69144ad3-7723-478e-966d-8e0a2bbd994c","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-05T09:40:00.219Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"69144ad3-7723-478e-966d-8e0a2bbd994c","name":"stop-mmlfk","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:40:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-mmlfk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:40:00.222Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"a19a95e1-8ae2-478c-af18-43db47ec1853","name":"stop-mmlfk-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:40:00.227Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"e3e0ce89-8a05-40d0-b50b-79b2b466bbfb","name":"stop-mmlfk-head"}
{"level":"info","ts":"2026-06-05T09:40:00.246Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"69144ad3-7723-478e-966d-8e0a2bbd994c"}
{"level":"error","ts":"2026-06-05T09:40:00.246Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"69144ad3-7723-478e-966d-8e0a2bbd994c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-mmlfk not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:40:00.246Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"da63594c-2a59-4872-b076-662859af8d4c"}
{"level":"info","ts":"2026-06-05T09:40:00.246Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"da63594c-2a59-4872-b076-662859af8d4c"}
{"level":"info","ts":"2026-06-05T09:40:00.246Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"da63594c-2a59-4872-b076-662859af8d4c","1 head service found":"stop-mmlfk-head-svc"}
{"level":"info","ts":"2026-06-05T09:40:00.246Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"da63594c-2a59-4872-b076-662859af8d4c","error":"head certificate secret ray-head-secret-stop-mmlfk not found"}
{"level":"error","ts":"2026-06-05T09:40:00.246Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"da63594c-2a59-4872-b076-662859af8d4c","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-mmlfk not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:40:00.246Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"da63594c-2a59-4872-b076-662859af8d4c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-mmlfk not found"}
{"level":"info","ts":"2026-06-05T09:40:00.246Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"da63594c-2a59-4872-b076-662859af8d4c","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"stop-mmlfk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:40:00.246Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"da63594c-2a59-4872-b076-662859af8d4c","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"stop-mmlfk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:40:00.246Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"da63594c-2a59-4872-b076-662859af8d4c","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"stop-mmlfk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:40:00.246Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"da63594c-2a59-4872-b076-662859af8d4c","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-05T09:40:00.246Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"da63594c-2a59-4872-b076-662859af8d4c","name":"stop-mmlfk","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:40:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-mmlfk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:40:00.306Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"cd0d3285-dfb4-4cec-b053-850408068de6","rayCluster":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:00.306Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"bfa3f97a-700b-409f-a8bb-84edd85ef0b7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:40:00.306Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"bfa3f97a-700b-409f-a8bb-84edd85ef0b7","name":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:00.306Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"bfa3f97a-700b-409f-a8bb-84edd85ef0b7","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:40:00.306Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"a19a95e1-8ae2-478c-af18-43db47ec1853","name":"test-ns-l4cdh-stop-mmlfk","operation":"created"}
{"level":"info","ts":"2026-06-05T09:40:00.306Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"bfa3f97a-700b-409f-a8bb-84edd85ef0b7"}
{"level":"info","ts":"2026-06-05T09:40:00.306Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"bfa3f97a-700b-409f-a8bb-84edd85ef0b7","RayCluster":"stop-mmlfk","State":""}
{"level":"info","ts":"2026-06-05T09:40:00.310Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"e3e0ce89-8a05-40d0-b50b-79b2b466bbfb","name":"stop-mmlfk-workers"}
{"level":"info","ts":"2026-06-05T09:40:00.310Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"e3e0ce89-8a05-40d0-b50b-79b2b466bbfb","cluster":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:00.310Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"6ae9bbd7-f538-4752-9e2f-04715e7f6468","cluster":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:00.311Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"a19a95e1-8ae2-478c-af18-43db47ec1853","name":"kube-rbac-proxy-config-stop-mmlfk","operation":"created"}
{"level":"info","ts":"2026-06-05T09:40:00.312Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"da63594c-2a59-4872-b076-662859af8d4c","name":"stop-mmlfk","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-mmlfk\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-l4cdh","name":"stop-mmlfk"}}
{"level":"info","ts":"2026-06-05T09:40:00.312Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"da63594c-2a59-4872-b076-662859af8d4c"}
{"level":"error","ts":"2026-06-05T09:40:00.312Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"da63594c-2a59-4872-b076-662859af8d4c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-mmlfk not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:40:00.313Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"54983e1f-41f1-4907-9041-0bbaf06d6bd6"}
{"level":"info","ts":"2026-06-05T09:40:00.313Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"54983e1f-41f1-4907-9041-0bbaf06d6bd6"}
{"level":"info","ts":"2026-06-05T09:40:00.313Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"54983e1f-41f1-4907-9041-0bbaf06d6bd6","1 head service found":"stop-mmlfk-head-svc"}
{"level":"info","ts":"2026-06-05T09:40:00.313Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"54983e1f-41f1-4907-9041-0bbaf06d6bd6","error":"head certificate secret ray-head-secret-stop-mmlfk not found"}
{"level":"error","ts":"2026-06-05T09:40:00.313Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"54983e1f-41f1-4907-9041-0bbaf06d6bd6","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-mmlfk not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:40:00.313Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"54983e1f-41f1-4907-9041-0bbaf06d6bd6","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-mmlfk not found"}
{"level":"info","ts":"2026-06-05T09:40:00.313Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"54983e1f-41f1-4907-9041-0bbaf06d6bd6","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"stop-mmlfk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:40:00.313Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"54983e1f-41f1-4907-9041-0bbaf06d6bd6","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"stop-mmlfk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:40:00.313Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"54983e1f-41f1-4907-9041-0bbaf06d6bd6","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"stop-mmlfk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:40:00.313Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"54983e1f-41f1-4907-9041-0bbaf06d6bd6"}
{"level":"error","ts":"2026-06-05T09:40:00.313Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"54983e1f-41f1-4907-9041-0bbaf06d6bd6","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-mmlfk not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:40:00.317Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"cd0d3285-dfb4-4cec-b053-850408068de6"}
{"level":"info","ts":"2026-06-05T09:40:00.317Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"cd0d3285-dfb4-4cec-b053-850408068de6","rayCluster":"stop-mmlfk","podIPs":[]}
{"level":"error","ts":"2026-06-05T09:40:00.318Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"a19a95e1-8ae2-478c-af18-43db47ec1853","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"stop-mmlfk\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:40:00.318Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"a19a95e1-8ae2-478c-af18-43db47ec1853"}
{"level":"error","ts":"2026-06-05T09:40:00.318Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"a19a95e1-8ae2-478c-af18-43db47ec1853","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"stop-mmlfk\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:40:00.318Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"b6c5b2ab-80c0-4d4b-b5e1-d03e225132aa","namespacedName":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:40:00.318Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"b6c5b2ab-80c0-4d4b-b5e1-d03e225132aa","mode":"IntegratedOAuth","cluster":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:00.318Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"b6c5b2ab-80c0-4d4b-b5e1-d03e225132aa","cluster":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:00.405Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"6ae9bbd7-f538-4752-9e2f-04715e7f6468","name":"stop-mmlfk-head"}
{"level":"info","ts":"2026-06-05T09:40:00.405Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"a6dc0b5e-e251-4318-bc7e-d8d2c33f28ea"}
{"level":"info","ts":"2026-06-05T09:40:00.405Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"a6dc0b5e-e251-4318-bc7e-d8d2c33f28ea"}
{"level":"info","ts":"2026-06-05T09:40:00.405Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"b6c5b2ab-80c0-4d4b-b5e1-d03e225132aa","cluster":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:00.406Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"a6dc0b5e-e251-4318-bc7e-d8d2c33f28ea","1 head service found":"stop-mmlfk-head-svc"}
{"level":"info","ts":"2026-06-05T09:40:00.406Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"a6dc0b5e-e251-4318-bc7e-d8d2c33f28ea","error":"head certificate secret ray-head-secret-stop-mmlfk not found"}
{"level":"info","ts":"2026-06-05T09:40:00.406Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"015eb792-abd1-48c5-9e3a-7a10a38ae2c0","namespacedName":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:40:00.406Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"cd0d3285-dfb4-4cec-b053-850408068de6"}
{"level":"info","ts":"2026-06-05T09:40:00.406Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"015eb792-abd1-48c5-9e3a-7a10a38ae2c0","mode":"IntegratedOAuth","cluster":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:00.406Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"015eb792-abd1-48c5-9e3a-7a10a38ae2c0","cluster":"stop-mmlfk"}
{"level":"error","ts":"2026-06-05T09:40:00.406Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"a6dc0b5e-e251-4318-bc7e-d8d2c33f28ea","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-mmlfk not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:40:00.406Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"a6dc0b5e-e251-4318-bc7e-d8d2c33f28ea","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-mmlfk not found"}
{"level":"info","ts":"2026-06-05T09:40:00.406Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"a6dc0b5e-e251-4318-bc7e-d8d2c33f28ea","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"stop-mmlfk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:40:00.406Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"cd0d3285-dfb4-4cec-b053-850408068de6","rayCluster":"stop-mmlfk","podIPs":[]}
{"level":"info","ts":"2026-06-05T09:40:00.406Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"a6dc0b5e-e251-4318-bc7e-d8d2c33f28ea","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"stop-mmlfk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:40:00.406Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"a6dc0b5e-e251-4318-bc7e-d8d2c33f28ea","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"stop-mmlfk","ray.io/node-type":"head"}]}
{"level":"error","ts":"2026-06-05T09:40:00.406Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"cd0d3285-dfb4-4cec-b053-850408068de6","error":"Certificate.cert-manager.io \"ray-worker-cert-stop-mmlfk\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:40:00.406Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"cd0d3285-dfb4-4cec-b053-850408068de6"}
{"level":"error","ts":"2026-06-05T09:40:00.406Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"cd0d3285-dfb4-4cec-b053-850408068de6","error":"Certificate.cert-manager.io \"ray-worker-cert-stop-mmlfk\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:40:00.406Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"44cc1524-4ddc-443c-b58e-5d2e184393cd","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:40:00.406Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"44cc1524-4ddc-443c-b58e-5d2e184393cd","name":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:00.406Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"d8a38c7b-1480-46af-923a-50204051179c","rayCluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:40:00.406Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"44cc1524-4ddc-443c-b58e-5d2e184393cd","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:40:00.406Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"a6dc0b5e-e251-4318-bc7e-d8d2c33f28ea"}
{"level":"error","ts":"2026-06-05T09:40:00.406Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"a6dc0b5e-e251-4318-bc7e-d8d2c33f28ea","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-mmlfk not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:40:00.406Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"d8a38c7b-1480-46af-923a-50204051179c","pod":"fail-d7vcr-head-8br8s","ip":"10.132.0.16"}
{"level":"info","ts":"2026-06-05T09:40:00.406Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"d8a38c7b-1480-46af-923a-50204051179c","pod":"fail-d7vcr-small-group-worker-crtnw","ip":"10.133.0.36"}
{"level":"info","ts":"2026-06-05T09:40:00.406Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"44cc1524-4ddc-443c-b58e-5d2e184393cd"}
{"level":"info","ts":"2026-06-05T09:40:00.406Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"44cc1524-4ddc-443c-b58e-5d2e184393cd","RayCluster":"stop-mmlfk","State":""}
{"level":"info","ts":"2026-06-05T09:40:00.406Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"d8a38c7b-1480-46af-923a-50204051179c","rayCluster":"fail-d7vcr","podIPs":["10.132.0.16","10.133.0.36"]}
{"level":"info","ts":"2026-06-05T09:40:00.406Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"ed53d014-1155-49ff-b95b-825da7de3a26","rayCluster":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:00.406Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"ed53d014-1155-49ff-b95b-825da7de3a26"}
{"level":"info","ts":"2026-06-05T09:40:00.412Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"c5354e21-cabe-4a5f-98d7-c0f33a3845be","rayCluster":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:00.412Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"c5354e21-cabe-4a5f-98d7-c0f33a3845be"}
{"level":"info","ts":"2026-06-05T09:40:00.417Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"015eb792-abd1-48c5-9e3a-7a10a38ae2c0","cluster":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:00.420Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"6ae9bbd7-f538-4752-9e2f-04715e7f6468","name":"stop-mmlfk-workers"}
{"level":"info","ts":"2026-06-05T09:40:00.420Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"6ae9bbd7-f538-4752-9e2f-04715e7f6468","cluster":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:00.420Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"2dce1886-ea1a-44f0-aaa2-6e0afd22d508","cluster":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:00.431Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"2dce1886-ea1a-44f0-aaa2-6e0afd22d508","name":"stop-mmlfk-head"}
{"level":"info","ts":"2026-06-05T09:40:00.505Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"3f7da784-451c-40e5-a3ed-a6ba388eeba8"}
{"level":"info","ts":"2026-06-05T09:40:00.506Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"3f7da784-451c-40e5-a3ed-a6ba388eeba8"}
{"level":"info","ts":"2026-06-05T09:40:00.506Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"3f7da784-451c-40e5-a3ed-a6ba388eeba8","1 head service found":"stop-mmlfk-head-svc"}
{"level":"info","ts":"2026-06-05T09:40:00.506Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"3f7da784-451c-40e5-a3ed-a6ba388eeba8","error":"head certificate secret ray-head-secret-stop-mmlfk not found"}
{"level":"error","ts":"2026-06-05T09:40:00.506Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"3f7da784-451c-40e5-a3ed-a6ba388eeba8","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-mmlfk not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:40:00.506Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"3f7da784-451c-40e5-a3ed-a6ba388eeba8","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-mmlfk not found"}
{"level":"info","ts":"2026-06-05T09:40:00.506Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"3f7da784-451c-40e5-a3ed-a6ba388eeba8","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"stop-mmlfk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:40:00.506Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"3f7da784-451c-40e5-a3ed-a6ba388eeba8","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"stop-mmlfk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:40:00.506Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"3f7da784-451c-40e5-a3ed-a6ba388eeba8","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"stop-mmlfk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:40:00.506Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"3f7da784-451c-40e5-a3ed-a6ba388eeba8"}
{"level":"error","ts":"2026-06-05T09:40:00.506Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"3f7da784-451c-40e5-a3ed-a6ba388eeba8","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-mmlfk not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:40:00.510Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"2dce1886-ea1a-44f0-aaa2-6e0afd22d508","name":"stop-mmlfk-workers"}
{"level":"info","ts":"2026-06-05T09:40:00.510Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"2dce1886-ea1a-44f0-aaa2-6e0afd22d508","cluster":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:00.666Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"3aee3e94-0de9-40ae-8a03-d3bb4ea2dd3f"}
{"level":"info","ts":"2026-06-05T09:40:00.666Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"3aee3e94-0de9-40ae-8a03-d3bb4ea2dd3f"}
{"level":"info","ts":"2026-06-05T09:40:00.666Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"3aee3e94-0de9-40ae-8a03-d3bb4ea2dd3f","1 head service found":"stop-mmlfk-head-svc"}
{"level":"info","ts":"2026-06-05T09:40:00.666Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"3aee3e94-0de9-40ae-8a03-d3bb4ea2dd3f","error":"head certificate secret ray-head-secret-stop-mmlfk not found"}
{"level":"error","ts":"2026-06-05T09:40:00.666Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"3aee3e94-0de9-40ae-8a03-d3bb4ea2dd3f","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-mmlfk not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:40:00.667Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"3aee3e94-0de9-40ae-8a03-d3bb4ea2dd3f","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-mmlfk not found"}
{"level":"info","ts":"2026-06-05T09:40:00.667Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"3aee3e94-0de9-40ae-8a03-d3bb4ea2dd3f","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"stop-mmlfk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:40:00.667Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"3aee3e94-0de9-40ae-8a03-d3bb4ea2dd3f","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"stop-mmlfk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:40:00.667Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"3aee3e94-0de9-40ae-8a03-d3bb4ea2dd3f","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"stop-mmlfk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:40:00.667Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"3aee3e94-0de9-40ae-8a03-d3bb4ea2dd3f"}
{"level":"error","ts":"2026-06-05T09:40:00.667Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"3aee3e94-0de9-40ae-8a03-d3bb4ea2dd3f","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-mmlfk not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:40:00.987Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"6274c74d-23b8-4ca9-a104-e39c4a03e1ea"}
{"level":"info","ts":"2026-06-05T09:40:00.987Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"6274c74d-23b8-4ca9-a104-e39c4a03e1ea"}
{"level":"info","ts":"2026-06-05T09:40:00.987Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"6274c74d-23b8-4ca9-a104-e39c4a03e1ea","1 head service found":"stop-mmlfk-head-svc"}
{"level":"info","ts":"2026-06-05T09:40:00.987Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"6274c74d-23b8-4ca9-a104-e39c4a03e1ea","error":"head certificate secret ray-head-secret-stop-mmlfk not found"}
{"level":"error","ts":"2026-06-05T09:40:00.987Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"6274c74d-23b8-4ca9-a104-e39c4a03e1ea","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-mmlfk not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:40:00.987Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"6274c74d-23b8-4ca9-a104-e39c4a03e1ea","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-mmlfk not found"}
{"level":"info","ts":"2026-06-05T09:40:00.987Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"6274c74d-23b8-4ca9-a104-e39c4a03e1ea","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"stop-mmlfk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:40:00.987Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"6274c74d-23b8-4ca9-a104-e39c4a03e1ea","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"stop-mmlfk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:40:00.987Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"6274c74d-23b8-4ca9-a104-e39c4a03e1ea","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"stop-mmlfk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:40:00.987Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"6274c74d-23b8-4ca9-a104-e39c4a03e1ea"}
{"level":"error","ts":"2026-06-05T09:40:00.987Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"6274c74d-23b8-4ca9-a104-e39c4a03e1ea","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-mmlfk not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:40:01.628Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"74e7306a-384e-4ce9-b715-1d4be296d77d"}
{"level":"info","ts":"2026-06-05T09:40:01.628Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"74e7306a-384e-4ce9-b715-1d4be296d77d"}
{"level":"info","ts":"2026-06-05T09:40:01.628Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"74e7306a-384e-4ce9-b715-1d4be296d77d","1 head service found":"stop-mmlfk-head-svc"}
{"level":"info","ts":"2026-06-05T09:40:01.628Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"74e7306a-384e-4ce9-b715-1d4be296d77d","error":"head certificate secret ray-head-secret-stop-mmlfk not found"}
{"level":"error","ts":"2026-06-05T09:40:01.628Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"74e7306a-384e-4ce9-b715-1d4be296d77d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-mmlfk not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:40:01.628Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"74e7306a-384e-4ce9-b715-1d4be296d77d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-mmlfk not found"}
{"level":"info","ts":"2026-06-05T09:40:01.628Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"74e7306a-384e-4ce9-b715-1d4be296d77d","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"stop-mmlfk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:40:01.629Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"74e7306a-384e-4ce9-b715-1d4be296d77d","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"stop-mmlfk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:40:01.629Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"74e7306a-384e-4ce9-b715-1d4be296d77d","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"stop-mmlfk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:40:01.629Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"74e7306a-384e-4ce9-b715-1d4be296d77d"}
{"level":"error","ts":"2026-06-05T09:40:01.629Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"74e7306a-384e-4ce9-b715-1d4be296d77d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-mmlfk not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:40:02.909Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"e8162787-e949-40fc-b47f-b23f7e5d9573"}
{"level":"info","ts":"2026-06-05T09:40:02.909Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"e8162787-e949-40fc-b47f-b23f7e5d9573"}
{"level":"info","ts":"2026-06-05T09:40:02.909Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"e8162787-e949-40fc-b47f-b23f7e5d9573","1 head service found":"stop-mmlfk-head-svc"}
{"level":"info","ts":"2026-06-05T09:40:02.909Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"e8162787-e949-40fc-b47f-b23f7e5d9573"}
{"level":"info","ts":"2026-06-05T09:40:02.909Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"e8162787-e949-40fc-b47f-b23f7e5d9573","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:40:02.909Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"e8162787-e949-40fc-b47f-b23f7e5d9573"}
{"level":"info","ts":"2026-06-05T09:40:02.909Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"e8162787-e949-40fc-b47f-b23f7e5d9573","mode":"IntegratedOAuth","cluster":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:02.909Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"e8162787-e949-40fc-b47f-b23f7e5d9573","cluster":"stop-mmlfk","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:40:02.909Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"e8162787-e949-40fc-b47f-b23f7e5d9573","cluster":"stop-mmlfk","authType":"IntegratedOAuth","serviceAccount":"stop-mmlfk-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-05T09:40:02.909Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"e8162787-e949-40fc-b47f-b23f7e5d9573","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"stop-mmlfk","ray.io/group":"headgroup","ray.io/identifier":"stop-mmlfk-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-05T09:40:02.909Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"e8162787-e949-40fc-b47f-b23f7e5d9573","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"stop-mmlfk-head-svc.test-ns-l4cdh.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-05T09:40:02.909Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"e8162787-e949-40fc-b47f-b23f7e5d9573","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=stop-mmlfk-head-svc.test-ns-l4cdh.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-05T09:40:02.909Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"e8162787-e949-40fc-b47f-b23f7e5d9573","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=stop-mmlfk-head-svc.test-ns-l4cdh.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-05T09:40:02.910Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"e8162787-e949-40fc-b47f-b23f7e5d9573","enabled":true}
{"level":"info","ts":"2026-06-05T09:40:03.006Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"e8162787-e949-40fc-b47f-b23f7e5d9573","name":"stop-mmlfk-head-7pzr6"}
{"level":"info","ts":"2026-06-05T09:40:03.006Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"e8162787-e949-40fc-b47f-b23f7e5d9573","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:40:03.049Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"57d0aae9-2083-4746-9505-c550d3f52cab","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:40:03.049Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"57d0aae9-2083-4746-9505-c550d3f52cab","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-05 09:40:00 +0000 UTC","Now":"2026-06-05T09:40:03.049Z","ShutdownTime":"2026-06-05T09:40:00.000Z"}
{"level":"info","ts":"2026-06-05T09:40:03.105Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"e8162787-e949-40fc-b47f-b23f7e5d9573","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:40:03.105Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"e8162787-e949-40fc-b47f-b23f7e5d9573","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-05T09:40:03.105Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"e8162787-e949-40fc-b47f-b23f7e5d9573","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-05T09:40:03.105Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"e8162787-e949-40fc-b47f-b23f7e5d9573","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-05T09:40:03.105Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"e8162787-e949-40fc-b47f-b23f7e5d9573"}
{"level":"info","ts":"2026-06-05T09:40:03.106Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"e8162787-e949-40fc-b47f-b23f7e5d9573","nodeType":"worker","rayStartParams":{"address":"stop-mmlfk-head-svc.test-ns-l4cdh.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-05T09:40:03.106Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"e8162787-e949-40fc-b47f-b23f7e5d9573","rayStartCmd":"ray start  --address=stop-mmlfk-head-svc.test-ns-l4cdh.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:40:03.106Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"e8162787-e949-40fc-b47f-b23f7e5d9573","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=stop-mmlfk-head-svc.test-ns-l4cdh.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:40:03.106Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"e8162787-e949-40fc-b47f-b23f7e5d9573","enabled":true}
{"level":"info","ts":"2026-06-05T09:40:03.121Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"e8162787-e949-40fc-b47f-b23f7e5d9573","name":"stop-mmlfk-small-group-worker-26zt6"}
{"level":"info","ts":"2026-06-05T09:40:03.121Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"e8162787-e949-40fc-b47f-b23f7e5d9573","podName":"stop-mmlfk-head-7pzr6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:40:03.121Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"e8162787-e949-40fc-b47f-b23f7e5d9573","podName":"stop-mmlfk-head-7pzr6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:40:03.121Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"e8162787-e949-40fc-b47f-b23f7e5d9573","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"stop-mmlfk-head-svc"},"newHead":{"podName":"stop-mmlfk-head-7pzr6","serviceName":"stop-mmlfk-head-svc"}}
{"level":"info","ts":"2026-06-05T09:40:03.121Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"e8162787-e949-40fc-b47f-b23f7e5d9573","name":"stop-mmlfk","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:40:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-mmlfk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:40:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-mmlfk-head-7pzr6","serviceName":"stop-mmlfk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:40:03.135Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"3c999561-6ce1-4772-9dcc-532250a02a36"}
{"level":"info","ts":"2026-06-05T09:40:03.135Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"3c999561-6ce1-4772-9dcc-532250a02a36"}
{"level":"info","ts":"2026-06-05T09:40:03.135Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"3c999561-6ce1-4772-9dcc-532250a02a36","1 head service found":"stop-mmlfk-head-svc"}
{"level":"info","ts":"2026-06-05T09:40:03.136Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"1a7aeb85-647b-4dda-b640-ad858a8faedc","rayCluster":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:03.136Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"3c999561-6ce1-4772-9dcc-532250a02a36","Found 1 head Pod":"stop-mmlfk-head-7pzr6","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:40:03.136Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"3c999561-6ce1-4772-9dcc-532250a02a36","head Pod":"stop-mmlfk-head-7pzr6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-mmlfk-head-7pzr6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:40:03.136Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"3c999561-6ce1-4772-9dcc-532250a02a36","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:40:03.136Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"8f398e04-cc4a-4713-bc4e-7a9bbb70f705","cluster":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:03.136Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"3c999561-6ce1-4772-9dcc-532250a02a36","worker Pod":"stop-mmlfk-small-group-worker-26zt6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-mmlfk-small-group-worker-26zt6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:40:03.136Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"1a7aeb85-647b-4dda-b640-ad858a8faedc","rayCluster":"stop-mmlfk","podIPs":[]}
{"level":"info","ts":"2026-06-05T09:40:03.136Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"3c999561-6ce1-4772-9dcc-532250a02a36","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:40:03.136Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"3c999561-6ce1-4772-9dcc-532250a02a36","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:40:03.136Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"3c999561-6ce1-4772-9dcc-532250a02a36","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:40:03.136Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"fa1ab6da-ad86-42cb-8f4f-b403f3fc11dc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:40:03.136Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"fa1ab6da-ad86-42cb-8f4f-b403f3fc11dc","name":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:03.136Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"fa1ab6da-ad86-42cb-8f4f-b403f3fc11dc","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:40:03.136Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"3c999561-6ce1-4772-9dcc-532250a02a36","podName":"stop-mmlfk-head-7pzr6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:40:03.136Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"3c999561-6ce1-4772-9dcc-532250a02a36","podName":"stop-mmlfk-head-7pzr6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:40:03.136Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"fa1ab6da-ad86-42cb-8f4f-b403f3fc11dc"}
{"level":"info","ts":"2026-06-05T09:40:03.136Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"fa1ab6da-ad86-42cb-8f4f-b403f3fc11dc","RayCluster":"stop-mmlfk","State":""}
{"level":"info","ts":"2026-06-05T09:40:03.136Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"3c999561-6ce1-4772-9dcc-532250a02a36","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"stop-mmlfk-head-svc"},"newHead":{"podName":"stop-mmlfk-head-7pzr6","serviceName":"stop-mmlfk-head-svc"}}
{"level":"info","ts":"2026-06-05T09:40:03.136Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"3c999561-6ce1-4772-9dcc-532250a02a36","name":"stop-mmlfk","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:40:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-mmlfk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:40:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-mmlfk-head-7pzr6","serviceName":"stop-mmlfk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:40:03.145Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"3c999561-6ce1-4772-9dcc-532250a02a36","name":"stop-mmlfk","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-mmlfk\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-l4cdh","name":"stop-mmlfk"}}
{"level":"info","ts":"2026-06-05T09:40:03.145Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"3c999561-6ce1-4772-9dcc-532250a02a36"}
{"level":"error","ts":"2026-06-05T09:40:03.145Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"3c999561-6ce1-4772-9dcc-532250a02a36","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-mmlfk\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:40:03.205Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"d6a85baf-480e-4f43-8d85-3ce7aafa0ec5"}
{"level":"info","ts":"2026-06-05T09:40:03.205Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"437ee66e-1ba5-4122-993d-53786d7cf0aa","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:40:03.206Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"437ee66e-1ba5-4122-993d-53786d7cf0aa","name":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:03.206Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"437ee66e-1ba5-4122-993d-53786d7cf0aa","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:40:03.206Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"d6a85baf-480e-4f43-8d85-3ce7aafa0ec5"}
{"level":"info","ts":"2026-06-05T09:40:03.206Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"d6a85baf-480e-4f43-8d85-3ce7aafa0ec5","1 head service found":"stop-mmlfk-head-svc"}
{"level":"info","ts":"2026-06-05T09:40:03.206Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"437ee66e-1ba5-4122-993d-53786d7cf0aa"}
{"level":"info","ts":"2026-06-05T09:40:03.206Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"437ee66e-1ba5-4122-993d-53786d7cf0aa","RayCluster":"stop-mmlfk","State":""}
{"level":"info","ts":"2026-06-05T09:40:03.206Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"d6a85baf-480e-4f43-8d85-3ce7aafa0ec5","Found 1 head Pod":"stop-mmlfk-head-7pzr6","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:40:03.206Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"d6a85baf-480e-4f43-8d85-3ce7aafa0ec5","head Pod":"stop-mmlfk-head-7pzr6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-mmlfk-head-7pzr6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:40:03.206Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"d6a85baf-480e-4f43-8d85-3ce7aafa0ec5","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:40:03.206Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"d6a85baf-480e-4f43-8d85-3ce7aafa0ec5","worker Pod":"stop-mmlfk-small-group-worker-26zt6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-mmlfk-small-group-worker-26zt6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:40:03.206Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"d6a85baf-480e-4f43-8d85-3ce7aafa0ec5","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:40:03.206Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"d6a85baf-480e-4f43-8d85-3ce7aafa0ec5","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:40:03.206Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"d6a85baf-480e-4f43-8d85-3ce7aafa0ec5","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:40:03.206Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"8f398e04-cc4a-4713-bc4e-7a9bbb70f705","name":"stop-mmlfk-head"}
{"level":"info","ts":"2026-06-05T09:40:03.206Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"d6a85baf-480e-4f43-8d85-3ce7aafa0ec5","podName":"stop-mmlfk-head-7pzr6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:40:03.206Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"d6a85baf-480e-4f43-8d85-3ce7aafa0ec5","podName":"stop-mmlfk-head-7pzr6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:40:03.206Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"d6a85baf-480e-4f43-8d85-3ce7aafa0ec5","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:40:03.206Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"d6a85baf-480e-4f43-8d85-3ce7aafa0ec5","seconds":300}
{"level":"info","ts":"2026-06-05T09:40:03.206Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"ad4ebad2-7e80-48e4-9bf2-2efc1331a715"}
{"level":"info","ts":"2026-06-05T09:40:03.206Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"ad4ebad2-7e80-48e4-9bf2-2efc1331a715"}
{"level":"info","ts":"2026-06-05T09:40:03.206Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"ad4ebad2-7e80-48e4-9bf2-2efc1331a715","1 head service found":"stop-mmlfk-head-svc"}
{"level":"info","ts":"2026-06-05T09:40:03.206Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"ad4ebad2-7e80-48e4-9bf2-2efc1331a715","Found 1 head Pod":"stop-mmlfk-head-7pzr6","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:40:03.206Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"ad4ebad2-7e80-48e4-9bf2-2efc1331a715","head Pod":"stop-mmlfk-head-7pzr6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-mmlfk-head-7pzr6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:40:03.206Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"ad4ebad2-7e80-48e4-9bf2-2efc1331a715","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:40:03.206Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"ad4ebad2-7e80-48e4-9bf2-2efc1331a715","worker Pod":"stop-mmlfk-small-group-worker-26zt6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-mmlfk-small-group-worker-26zt6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:40:03.206Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"ad4ebad2-7e80-48e4-9bf2-2efc1331a715","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:40:03.206Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"ad4ebad2-7e80-48e4-9bf2-2efc1331a715","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:40:03.206Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"ad4ebad2-7e80-48e4-9bf2-2efc1331a715","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:40:03.206Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"ad4ebad2-7e80-48e4-9bf2-2efc1331a715","podName":"stop-mmlfk-head-7pzr6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:40:03.206Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"ad4ebad2-7e80-48e4-9bf2-2efc1331a715","podName":"stop-mmlfk-head-7pzr6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:40:03.206Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"ad4ebad2-7e80-48e4-9bf2-2efc1331a715","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:40:03.206Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"ad4ebad2-7e80-48e4-9bf2-2efc1331a715","seconds":300}
{"level":"info","ts":"2026-06-05T09:40:03.218Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"8f398e04-cc4a-4713-bc4e-7a9bbb70f705","name":"stop-mmlfk-workers"}
{"level":"info","ts":"2026-06-05T09:40:03.218Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"8f398e04-cc4a-4713-bc4e-7a9bbb70f705","cluster":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:03.398Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"311d1304-6ba1-4ce9-9af8-43a3b3ccc21c"}
{"level":"info","ts":"2026-06-05T09:40:03.398Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"311d1304-6ba1-4ce9-9af8-43a3b3ccc21c"}
{"level":"info","ts":"2026-06-05T09:40:03.398Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"311d1304-6ba1-4ce9-9af8-43a3b3ccc21c","1 head service found":"stop-mmlfk-head-svc"}
{"level":"info","ts":"2026-06-05T09:40:03.398Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"311d1304-6ba1-4ce9-9af8-43a3b3ccc21c","Found 1 head Pod":"stop-mmlfk-head-7pzr6","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:40:03.398Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"311d1304-6ba1-4ce9-9af8-43a3b3ccc21c","head Pod":"stop-mmlfk-head-7pzr6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-mmlfk-head-7pzr6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:40:03.398Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"311d1304-6ba1-4ce9-9af8-43a3b3ccc21c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:40:03.398Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"311d1304-6ba1-4ce9-9af8-43a3b3ccc21c","worker Pod":"stop-mmlfk-small-group-worker-26zt6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-mmlfk-small-group-worker-26zt6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:40:03.398Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"311d1304-6ba1-4ce9-9af8-43a3b3ccc21c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:40:03.398Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"311d1304-6ba1-4ce9-9af8-43a3b3ccc21c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:40:03.398Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"311d1304-6ba1-4ce9-9af8-43a3b3ccc21c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:40:03.398Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"311d1304-6ba1-4ce9-9af8-43a3b3ccc21c","podName":"stop-mmlfk-head-7pzr6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:40:03.398Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"311d1304-6ba1-4ce9-9af8-43a3b3ccc21c","podName":"stop-mmlfk-head-7pzr6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:40:03.398Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"311d1304-6ba1-4ce9-9af8-43a3b3ccc21c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:40:03.398Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"311d1304-6ba1-4ce9-9af8-43a3b3ccc21c","seconds":300}
{"level":"info","ts":"2026-06-05T09:40:03.578Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"2e9897f6-29e9-4ed7-b00b-f12861b6ce4f"}
{"level":"info","ts":"2026-06-05T09:40:03.578Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"2e9897f6-29e9-4ed7-b00b-f12861b6ce4f"}
{"level":"info","ts":"2026-06-05T09:40:03.578Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"2e9897f6-29e9-4ed7-b00b-f12861b6ce4f","1 head service found":"stop-mmlfk-head-svc"}
{"level":"info","ts":"2026-06-05T09:40:03.578Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"2e9897f6-29e9-4ed7-b00b-f12861b6ce4f","Found 1 head Pod":"stop-mmlfk-head-7pzr6","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:40:03.578Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"2e9897f6-29e9-4ed7-b00b-f12861b6ce4f","head Pod":"stop-mmlfk-head-7pzr6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-mmlfk-head-7pzr6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:40:03.578Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"2e9897f6-29e9-4ed7-b00b-f12861b6ce4f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:40:03.578Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"2e9897f6-29e9-4ed7-b00b-f12861b6ce4f","worker Pod":"stop-mmlfk-small-group-worker-26zt6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-mmlfk-small-group-worker-26zt6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:40:03.578Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"2e9897f6-29e9-4ed7-b00b-f12861b6ce4f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:40:03.578Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"2e9897f6-29e9-4ed7-b00b-f12861b6ce4f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:40:03.578Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"2e9897f6-29e9-4ed7-b00b-f12861b6ce4f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:40:03.578Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"2e9897f6-29e9-4ed7-b00b-f12861b6ce4f","podName":"stop-mmlfk-head-7pzr6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:40:03.578Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"2e9897f6-29e9-4ed7-b00b-f12861b6ce4f","podName":"stop-mmlfk-head-7pzr6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:40:03.578Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"2e9897f6-29e9-4ed7-b00b-f12861b6ce4f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:40:03.578Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"2e9897f6-29e9-4ed7-b00b-f12861b6ce4f","seconds":300}
{"level":"info","ts":"2026-06-05T09:40:03.907Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"109e450f-be07-4ef0-b1a3-535dd76cabc2"}
{"level":"info","ts":"2026-06-05T09:40:03.907Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"109e450f-be07-4ef0-b1a3-535dd76cabc2"}
{"level":"info","ts":"2026-06-05T09:40:03.907Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"109e450f-be07-4ef0-b1a3-535dd76cabc2","1 head service found":"stop-mmlfk-head-svc"}
{"level":"info","ts":"2026-06-05T09:40:03.907Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"109e450f-be07-4ef0-b1a3-535dd76cabc2","Found 1 head Pod":"stop-mmlfk-head-7pzr6","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:40:03.907Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"109e450f-be07-4ef0-b1a3-535dd76cabc2","head Pod":"stop-mmlfk-head-7pzr6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-mmlfk-head-7pzr6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:40:03.907Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"109e450f-be07-4ef0-b1a3-535dd76cabc2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:40:03.907Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"109e450f-be07-4ef0-b1a3-535dd76cabc2","worker Pod":"stop-mmlfk-small-group-worker-26zt6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-mmlfk-small-group-worker-26zt6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:40:03.907Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"109e450f-be07-4ef0-b1a3-535dd76cabc2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:40:03.907Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"109e450f-be07-4ef0-b1a3-535dd76cabc2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:40:03.907Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"109e450f-be07-4ef0-b1a3-535dd76cabc2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:40:03.907Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"109e450f-be07-4ef0-b1a3-535dd76cabc2","podName":"stop-mmlfk-head-7pzr6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:40:03.907Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"109e450f-be07-4ef0-b1a3-535dd76cabc2","podName":"stop-mmlfk-head-7pzr6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:40:03.907Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"109e450f-be07-4ef0-b1a3-535dd76cabc2","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:40:03.907Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"109e450f-be07-4ef0-b1a3-535dd76cabc2","seconds":300}
{"level":"info","ts":"2026-06-05T09:40:04.310Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"ad5bfbc9-351d-496f-9782-64b0951dea10"}
{"level":"info","ts":"2026-06-05T09:40:04.311Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"ad5bfbc9-351d-496f-9782-64b0951dea10"}
{"level":"info","ts":"2026-06-05T09:40:04.311Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"ad5bfbc9-351d-496f-9782-64b0951dea10","1 head service found":"stop-mmlfk-head-svc"}
{"level":"info","ts":"2026-06-05T09:40:04.311Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"ad5bfbc9-351d-496f-9782-64b0951dea10","Found 1 head Pod":"stop-mmlfk-head-7pzr6","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:40:04.311Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"ad5bfbc9-351d-496f-9782-64b0951dea10","head Pod":"stop-mmlfk-head-7pzr6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-mmlfk-head-7pzr6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:40:04.311Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"ad5bfbc9-351d-496f-9782-64b0951dea10","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:40:04.311Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"ad5bfbc9-351d-496f-9782-64b0951dea10","worker Pod":"stop-mmlfk-small-group-worker-26zt6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-mmlfk-small-group-worker-26zt6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:40:04.311Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"ad5bfbc9-351d-496f-9782-64b0951dea10","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:40:04.311Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"ad5bfbc9-351d-496f-9782-64b0951dea10","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:40:04.311Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"ad5bfbc9-351d-496f-9782-64b0951dea10","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:40:04.311Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"ad5bfbc9-351d-496f-9782-64b0951dea10","podName":"stop-mmlfk-head-7pzr6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:40:04.311Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"ad5bfbc9-351d-496f-9782-64b0951dea10","podName":"stop-mmlfk-head-7pzr6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:40:04.311Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"ad5bfbc9-351d-496f-9782-64b0951dea10","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"stop-mmlfk-head-7pzr6","serviceName":"stop-mmlfk-head-svc"},"newHead":{"podIP":"10.133.0.37","serviceIP":"10.133.0.37","podName":"stop-mmlfk-head-7pzr6","serviceName":"stop-mmlfk-head-svc"}}
{"level":"info","ts":"2026-06-05T09:40:04.311Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"ad5bfbc9-351d-496f-9782-64b0951dea10","name":"stop-mmlfk","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:40:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-mmlfk-head-7pzr6","serviceName":"stop-mmlfk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:40:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.37","serviceIP":"10.133.0.37","podName":"stop-mmlfk-head-7pzr6","serviceName":"stop-mmlfk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:40:04.336Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"615da237-299d-463e-bc5e-4c982bafb227","cluster":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:04.336Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"6b979035-cc0e-4bbd-8ab0-403d49076193","rayCluster":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:04.336Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"4c4d1b1c-c0b0-4ec8-916e-58c909d999d5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:40:04.336Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"4c4d1b1c-c0b0-4ec8-916e-58c909d999d5","name":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:04.336Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"4c4d1b1c-c0b0-4ec8-916e-58c909d999d5","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:40:04.336Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"6b979035-cc0e-4bbd-8ab0-403d49076193","pod":"stop-mmlfk-head-7pzr6","ip":"10.133.0.37"}
{"level":"info","ts":"2026-06-05T09:40:04.336Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"6b979035-cc0e-4bbd-8ab0-403d49076193","pod":"stop-mmlfk-small-group-worker-26zt6","ip":"10.132.0.17"}
{"level":"info","ts":"2026-06-05T09:40:04.337Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"4c4d1b1c-c0b0-4ec8-916e-58c909d999d5"}
{"level":"info","ts":"2026-06-05T09:40:04.337Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"4c4d1b1c-c0b0-4ec8-916e-58c909d999d5","RayCluster":"stop-mmlfk","State":""}
{"level":"info","ts":"2026-06-05T09:40:04.348Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"6b979035-cc0e-4bbd-8ab0-403d49076193","certificate":"ray-head-cert-stop-mmlfk","dnsNames":["stop-mmlfk-head-svc","localhost","stop-mmlfk-head-svc.test-ns-l4cdh.svc","stop-mmlfk-head-svc.test-ns-l4cdh.svc.cluster.local"],"ipAddresses":["10.133.0.37","10.132.0.17","127.0.0.1"]}
{"level":"info","ts":"2026-06-05T09:40:04.350Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"615da237-299d-463e-bc5e-4c982bafb227","name":"stop-mmlfk-head"}
{"level":"info","ts":"2026-06-05T09:40:04.362Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"6b979035-cc0e-4bbd-8ab0-403d49076193","certificate":"ray-worker-cert-stop-mmlfk","dnsNames":["stop-mmlfk-worker-svc","localhost","stop-mmlfk-small-group","stop-mmlfk-worker-svc.test-ns-l4cdh.svc","stop-mmlfk-worker-svc.test-ns-l4cdh.svc.cluster.local","*.stop-mmlfk-worker-svc.test-ns-l4cdh.svc","*.stop-mmlfk-worker-svc.test-ns-l4cdh.svc.cluster.local","*-worker-*.test-ns-l4cdh.svc","*-worker-*.test-ns-l4cdh.svc.cluster.local"],"ipAddresses":["10.133.0.37","10.132.0.17","127.0.0.1"]}
{"level":"info","ts":"2026-06-05T09:40:04.362Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"6b979035-cc0e-4bbd-8ab0-403d49076193","rayCluster":"stop-mmlfk","podIPs":["10.133.0.37","10.132.0.17"]}
{"level":"info","ts":"2026-06-05T09:40:04.364Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"615da237-299d-463e-bc5e-4c982bafb227","name":"stop-mmlfk-workers"}
{"level":"info","ts":"2026-06-05T09:40:04.364Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"615da237-299d-463e-bc5e-4c982bafb227","cluster":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:06.206Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"a8711ac5-097c-4c8b-bdf2-e6083ab38e35","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:40:06.206Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"a8711ac5-097c-4c8b-bdf2-e6083ab38e35","name":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:06.206Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"a8711ac5-097c-4c8b-bdf2-e6083ab38e35","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:40:06.206Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"a8711ac5-097c-4c8b-bdf2-e6083ab38e35"}
{"level":"info","ts":"2026-06-05T09:40:06.206Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"a8711ac5-097c-4c8b-bdf2-e6083ab38e35","RayCluster":"stop-mmlfk","State":""}
{"level":"info","ts":"2026-06-05T09:40:06.336Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"20fca45a-76d9-49e6-bb40-9d749f898042"}
{"level":"info","ts":"2026-06-05T09:40:06.336Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"20fca45a-76d9-49e6-bb40-9d749f898042"}
{"level":"info","ts":"2026-06-05T09:40:06.337Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"20fca45a-76d9-49e6-bb40-9d749f898042","1 head service found":"stop-mmlfk-head-svc"}
{"level":"info","ts":"2026-06-05T09:40:06.337Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"20fca45a-76d9-49e6-bb40-9d749f898042","Found 1 head Pod":"stop-mmlfk-head-7pzr6","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:40:06.337Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"20fca45a-76d9-49e6-bb40-9d749f898042","head Pod":"stop-mmlfk-head-7pzr6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-mmlfk-head-7pzr6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:40:06.337Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"20fca45a-76d9-49e6-bb40-9d749f898042","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:40:06.337Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"20fca45a-76d9-49e6-bb40-9d749f898042","worker Pod":"stop-mmlfk-small-group-worker-26zt6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-mmlfk-small-group-worker-26zt6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:40:06.337Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"20fca45a-76d9-49e6-bb40-9d749f898042","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:40:06.337Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"20fca45a-76d9-49e6-bb40-9d749f898042","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:40:06.337Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"20fca45a-76d9-49e6-bb40-9d749f898042","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:40:06.337Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"20fca45a-76d9-49e6-bb40-9d749f898042","podName":"stop-mmlfk-head-7pzr6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:40:06.337Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"20fca45a-76d9-49e6-bb40-9d749f898042","podName":"stop-mmlfk-head-7pzr6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:40:06.337Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"20fca45a-76d9-49e6-bb40-9d749f898042","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:40:06.337Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"20fca45a-76d9-49e6-bb40-9d749f898042","seconds":300}
{"level":"info","ts":"2026-06-05T09:40:09.208Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"2c07406b-2fa0-4130-b77c-80686b7a6721","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:40:09.208Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"2c07406b-2fa0-4130-b77c-80686b7a6721","name":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:09.208Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"2c07406b-2fa0-4130-b77c-80686b7a6721","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:40:09.208Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"2c07406b-2fa0-4130-b77c-80686b7a6721"}
{"level":"info","ts":"2026-06-05T09:40:09.208Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"2c07406b-2fa0-4130-b77c-80686b7a6721","RayCluster":"stop-mmlfk","State":""}
{"level":"info","ts":"2026-06-05T09:40:09.331Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"494d13ac-dbab-4731-a643-96024c396fca"}
{"level":"info","ts":"2026-06-05T09:40:09.331Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"494d13ac-dbab-4731-a643-96024c396fca"}
{"level":"info","ts":"2026-06-05T09:40:09.331Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"494d13ac-dbab-4731-a643-96024c396fca","1 head service found":"stop-mmlfk-head-svc"}
{"level":"info","ts":"2026-06-05T09:40:09.331Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"494d13ac-dbab-4731-a643-96024c396fca","Found 1 head Pod":"stop-mmlfk-head-7pzr6","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:40:09.331Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"494d13ac-dbab-4731-a643-96024c396fca","head Pod":"stop-mmlfk-head-7pzr6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-mmlfk-head-7pzr6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:40:09.331Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"494d13ac-dbab-4731-a643-96024c396fca","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:40:09.331Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"494d13ac-dbab-4731-a643-96024c396fca","worker Pod":"stop-mmlfk-small-group-worker-26zt6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-mmlfk-small-group-worker-26zt6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:40:09.331Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"494d13ac-dbab-4731-a643-96024c396fca","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:40:09.331Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"494d13ac-dbab-4731-a643-96024c396fca","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:40:09.331Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"494d13ac-dbab-4731-a643-96024c396fca","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:40:09.331Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"494d13ac-dbab-4731-a643-96024c396fca","podName":"stop-mmlfk-small-group-worker-26zt6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:40:09.331Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"494d13ac-dbab-4731-a643-96024c396fca","podName":"stop-mmlfk-small-group-worker-26zt6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:40:09.331Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"494d13ac-dbab-4731-a643-96024c396fca","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:40:09.331Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"494d13ac-dbab-4731-a643-96024c396fca","seconds":300}
{"level":"info","ts":"2026-06-05T09:40:12.209Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"56ec8af8-1483-4e55-aa83-06f211a7385f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:40:12.209Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"56ec8af8-1483-4e55-aa83-06f211a7385f","name":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:12.209Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"56ec8af8-1483-4e55-aa83-06f211a7385f","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:40:12.209Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"56ec8af8-1483-4e55-aa83-06f211a7385f"}
{"level":"info","ts":"2026-06-05T09:40:12.209Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"56ec8af8-1483-4e55-aa83-06f211a7385f","RayCluster":"stop-mmlfk","State":""}
{"level":"info","ts":"2026-06-05T09:40:15.210Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"e4fedf19-0628-4160-ac0d-0e420f3cc2ce","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:40:15.210Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"e4fedf19-0628-4160-ac0d-0e420f3cc2ce","name":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:15.210Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"e4fedf19-0628-4160-ac0d-0e420f3cc2ce","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:40:15.210Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"e4fedf19-0628-4160-ac0d-0e420f3cc2ce"}
{"level":"info","ts":"2026-06-05T09:40:15.210Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"e4fedf19-0628-4160-ac0d-0e420f3cc2ce","RayCluster":"stop-mmlfk","State":""}
{"level":"info","ts":"2026-06-05T09:40:16.383Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"bab20c40-d19d-4dab-ad7a-08962b7ba6a0"}
{"level":"info","ts":"2026-06-05T09:40:16.383Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"bab20c40-d19d-4dab-ad7a-08962b7ba6a0"}
{"level":"info","ts":"2026-06-05T09:40:16.383Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"bab20c40-d19d-4dab-ad7a-08962b7ba6a0","1 head service found":"stop-mmlfk-head-svc"}
{"level":"info","ts":"2026-06-05T09:40:16.383Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"bab20c40-d19d-4dab-ad7a-08962b7ba6a0","Found 1 head Pod":"stop-mmlfk-head-7pzr6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:40:16.383Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"bab20c40-d19d-4dab-ad7a-08962b7ba6a0","head Pod":"stop-mmlfk-head-7pzr6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-mmlfk-head-7pzr6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:40:16.383Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"bab20c40-d19d-4dab-ad7a-08962b7ba6a0","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:40:16.383Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"bab20c40-d19d-4dab-ad7a-08962b7ba6a0","worker Pod":"stop-mmlfk-small-group-worker-26zt6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-mmlfk-small-group-worker-26zt6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:40:16.383Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"bab20c40-d19d-4dab-ad7a-08962b7ba6a0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:40:16.383Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"bab20c40-d19d-4dab-ad7a-08962b7ba6a0","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:40:16.383Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"bab20c40-d19d-4dab-ad7a-08962b7ba6a0","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:40:16.383Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"bab20c40-d19d-4dab-ad7a-08962b7ba6a0","podName":"stop-mmlfk-head-7pzr6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:40:02Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:40:16.383Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"bab20c40-d19d-4dab-ad7a-08962b7ba6a0","podName":"stop-mmlfk-head-7pzr6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:40:02Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:40:16.384Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"bab20c40-d19d-4dab-ad7a-08962b7ba6a0","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-05T09:40:16.384Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"bab20c40-d19d-4dab-ad7a-08962b7ba6a0","name":"stop-mmlfk","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:40:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.37","serviceIP":"10.133.0.37","podName":"stop-mmlfk-head-7pzr6","serviceName":"stop-mmlfk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:40:16Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.37","serviceIP":"10.133.0.37","podName":"stop-mmlfk-head-7pzr6","serviceName":"stop-mmlfk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:40:16.398Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"392459a9-a3fa-4d7c-a349-f3c02a2bf72f","rayCluster":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:16.398Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"841e100a-346b-43b0-b610-829f8fbb7e5d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:40:16.398Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"841e100a-346b-43b0-b610-829f8fbb7e5d","name":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:16.398Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"841e100a-346b-43b0-b610-829f8fbb7e5d","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:40:16.398Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"03f52063-4a4a-499d-94a1-e3da78aff4f2","cluster":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:16.398Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"392459a9-a3fa-4d7c-a349-f3c02a2bf72f","pod":"stop-mmlfk-head-7pzr6","ip":"10.133.0.37"}
{"level":"info","ts":"2026-06-05T09:40:16.398Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"392459a9-a3fa-4d7c-a349-f3c02a2bf72f","pod":"stop-mmlfk-small-group-worker-26zt6","ip":"10.132.0.17"}
{"level":"info","ts":"2026-06-05T09:40:16.398Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"392459a9-a3fa-4d7c-a349-f3c02a2bf72f","rayCluster":"stop-mmlfk","podIPs":["10.133.0.37","10.132.0.17"]}
{"level":"info","ts":"2026-06-05T09:40:16.398Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"841e100a-346b-43b0-b610-829f8fbb7e5d"}
{"level":"info","ts":"2026-06-05T09:40:16.398Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"841e100a-346b-43b0-b610-829f8fbb7e5d","RayCluster":"stop-mmlfk","State":""}
{"level":"info","ts":"2026-06-05T09:40:16.430Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"03f52063-4a4a-499d-94a1-e3da78aff4f2","name":"stop-mmlfk-head"}
{"level":"info","ts":"2026-06-05T09:40:16.441Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"03f52063-4a4a-499d-94a1-e3da78aff4f2","name":"stop-mmlfk-workers"}
{"level":"info","ts":"2026-06-05T09:40:16.441Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"03f52063-4a4a-499d-94a1-e3da78aff4f2","cluster":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:18.213Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"6f212b76-2f82-4555-8aaa-cd25e7466ff8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:40:18.213Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"6f212b76-2f82-4555-8aaa-cd25e7466ff8","name":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:18.213Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"6f212b76-2f82-4555-8aaa-cd25e7466ff8","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:40:18.213Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"6f212b76-2f82-4555-8aaa-cd25e7466ff8"}
{"level":"info","ts":"2026-06-05T09:40:18.213Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"6f212b76-2f82-4555-8aaa-cd25e7466ff8","RayCluster":"stop-mmlfk","State":""}
{"level":"info","ts":"2026-06-05T09:40:18.398Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"dcaa1e95-6260-46d9-bcd6-e1133258f9be"}
{"level":"info","ts":"2026-06-05T09:40:18.398Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"dcaa1e95-6260-46d9-bcd6-e1133258f9be"}
{"level":"info","ts":"2026-06-05T09:40:18.398Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"dcaa1e95-6260-46d9-bcd6-e1133258f9be","1 head service found":"stop-mmlfk-head-svc"}
{"level":"info","ts":"2026-06-05T09:40:18.398Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"dcaa1e95-6260-46d9-bcd6-e1133258f9be","Found 1 head Pod":"stop-mmlfk-head-7pzr6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:40:18.398Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"dcaa1e95-6260-46d9-bcd6-e1133258f9be","head Pod":"stop-mmlfk-head-7pzr6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-mmlfk-head-7pzr6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:40:18.398Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"dcaa1e95-6260-46d9-bcd6-e1133258f9be","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:40:18.398Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"dcaa1e95-6260-46d9-bcd6-e1133258f9be","worker Pod":"stop-mmlfk-small-group-worker-26zt6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-mmlfk-small-group-worker-26zt6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:40:18.398Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"dcaa1e95-6260-46d9-bcd6-e1133258f9be","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:40:18.398Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"dcaa1e95-6260-46d9-bcd6-e1133258f9be","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:40:18.398Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"dcaa1e95-6260-46d9-bcd6-e1133258f9be","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:40:18.398Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"dcaa1e95-6260-46d9-bcd6-e1133258f9be","podName":"stop-mmlfk-head-7pzr6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:40:02Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:40:18.398Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"dcaa1e95-6260-46d9-bcd6-e1133258f9be","podName":"stop-mmlfk-head-7pzr6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:40:02Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:40:18.398Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"dcaa1e95-6260-46d9-bcd6-e1133258f9be","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:40:18.398Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"dcaa1e95-6260-46d9-bcd6-e1133258f9be","seconds":300}
{"level":"info","ts":"2026-06-05T09:40:21.214Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"87e31015-173c-4625-9a96-45f36a3b4cdc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:40:21.214Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"87e31015-173c-4625-9a96-45f36a3b4cdc","name":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:21.214Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"87e31015-173c-4625-9a96-45f36a3b4cdc","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:40:21.214Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"87e31015-173c-4625-9a96-45f36a3b4cdc"}
{"level":"info","ts":"2026-06-05T09:40:21.214Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"87e31015-173c-4625-9a96-45f36a3b4cdc","RayCluster":"stop-mmlfk","State":""}
{"level":"info","ts":"2026-06-05T09:40:22.411Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"e30008d4-5479-4592-b503-733fb553e295"}
{"level":"info","ts":"2026-06-05T09:40:22.411Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"e30008d4-5479-4592-b503-733fb553e295"}
{"level":"info","ts":"2026-06-05T09:40:22.411Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"e30008d4-5479-4592-b503-733fb553e295","1 head service found":"stop-mmlfk-head-svc"}
{"level":"info","ts":"2026-06-05T09:40:22.411Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"e30008d4-5479-4592-b503-733fb553e295","Found 1 head Pod":"stop-mmlfk-head-7pzr6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:40:22.411Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"e30008d4-5479-4592-b503-733fb553e295","head Pod":"stop-mmlfk-head-7pzr6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-mmlfk-head-7pzr6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:40:22.411Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"e30008d4-5479-4592-b503-733fb553e295","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:40:22.411Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"e30008d4-5479-4592-b503-733fb553e295","worker Pod":"stop-mmlfk-small-group-worker-26zt6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-mmlfk-small-group-worker-26zt6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:40:22.411Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"e30008d4-5479-4592-b503-733fb553e295","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:40:22.411Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"e30008d4-5479-4592-b503-733fb553e295","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:40:22.411Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"e30008d4-5479-4592-b503-733fb553e295","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:40:22.412Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"e30008d4-5479-4592-b503-733fb553e295","podName":"stop-mmlfk-small-group-worker-26zt6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:40:22.412Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"e30008d4-5479-4592-b503-733fb553e295","podName":"stop-mmlfk-small-group-worker-26zt6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:40:22.412Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"e30008d4-5479-4592-b503-733fb553e295","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:40:22Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-05T09:40:22.412Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"e30008d4-5479-4592-b503-733fb553e295","name":"stop-mmlfk","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:40:16Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.37","serviceIP":"10.133.0.37","podName":"stop-mmlfk-head-7pzr6","serviceName":"stop-mmlfk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:40:22Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.37","serviceIP":"10.133.0.37","podName":"stop-mmlfk-head-7pzr6","serviceName":"stop-mmlfk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:40:22Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:40:22.425Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"343e7bc8-bb0b-4f4b-9422-964840a904e1","rayCluster":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:22.425Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"4b885ff0-2bac-442f-95ac-93484578e02a","cluster":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:22.425Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"343e7bc8-bb0b-4f4b-9422-964840a904e1","pod":"stop-mmlfk-small-group-worker-26zt6","ip":"10.132.0.17"}
{"level":"info","ts":"2026-06-05T09:40:22.425Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"343e7bc8-bb0b-4f4b-9422-964840a904e1","pod":"stop-mmlfk-head-7pzr6","ip":"10.133.0.37"}
{"level":"info","ts":"2026-06-05T09:40:22.425Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"49d13646-ef4e-496f-979d-abc8f681fa63","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:40:22.425Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"49d13646-ef4e-496f-979d-abc8f681fa63","name":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:22.425Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"49d13646-ef4e-496f-979d-abc8f681fa63","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:40:22.425Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"343e7bc8-bb0b-4f4b-9422-964840a904e1","rayCluster":"stop-mmlfk","podIPs":["10.132.0.17","10.133.0.37"]}
{"level":"info","ts":"2026-06-05T09:40:22.425Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"49d13646-ef4e-496f-979d-abc8f681fa63"}
{"level":"info","ts":"2026-06-05T09:40:22.425Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"49d13646-ef4e-496f-979d-abc8f681fa63","RayCluster":"stop-mmlfk","State":""}
{"level":"info","ts":"2026-06-05T09:40:22.434Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"4b885ff0-2bac-442f-95ac-93484578e02a","name":"stop-mmlfk-head"}
{"level":"info","ts":"2026-06-05T09:40:22.445Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"4b885ff0-2bac-442f-95ac-93484578e02a","name":"stop-mmlfk-workers"}
{"level":"info","ts":"2026-06-05T09:40:22.445Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"4b885ff0-2bac-442f-95ac-93484578e02a","cluster":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:24.216Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"5abb723b-688d-442d-9e7d-f9271b6c1977","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:40:24.216Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"5abb723b-688d-442d-9e7d-f9271b6c1977","name":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:24.216Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"5abb723b-688d-442d-9e7d-f9271b6c1977","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:40:24.216Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"5abb723b-688d-442d-9e7d-f9271b6c1977"}
{"level":"info","ts":"2026-06-05T09:40:24.216Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"5abb723b-688d-442d-9e7d-f9271b6c1977","RayCluster":"stop-mmlfk","State":""}
{"level":"info","ts":"2026-06-05T09:40:24.425Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"4bfdd7ab-97a1-4e5e-b6a6-c50477113096"}
{"level":"info","ts":"2026-06-05T09:40:24.425Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"4bfdd7ab-97a1-4e5e-b6a6-c50477113096"}
{"level":"info","ts":"2026-06-05T09:40:24.425Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"4bfdd7ab-97a1-4e5e-b6a6-c50477113096","1 head service found":"stop-mmlfk-head-svc"}
{"level":"info","ts":"2026-06-05T09:40:24.425Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"4bfdd7ab-97a1-4e5e-b6a6-c50477113096","Found 1 head Pod":"stop-mmlfk-head-7pzr6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:40:24.425Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"4bfdd7ab-97a1-4e5e-b6a6-c50477113096","head Pod":"stop-mmlfk-head-7pzr6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-mmlfk-head-7pzr6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:40:24.425Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"4bfdd7ab-97a1-4e5e-b6a6-c50477113096","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:40:24.425Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"4bfdd7ab-97a1-4e5e-b6a6-c50477113096","worker Pod":"stop-mmlfk-small-group-worker-26zt6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-mmlfk-small-group-worker-26zt6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:40:24.425Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"4bfdd7ab-97a1-4e5e-b6a6-c50477113096","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:40:24.425Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"4bfdd7ab-97a1-4e5e-b6a6-c50477113096","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:40:24.425Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"4bfdd7ab-97a1-4e5e-b6a6-c50477113096","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:40:24.425Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"4bfdd7ab-97a1-4e5e-b6a6-c50477113096","podName":"stop-mmlfk-small-group-worker-26zt6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:40:24.425Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"4bfdd7ab-97a1-4e5e-b6a6-c50477113096","podName":"stop-mmlfk-small-group-worker-26zt6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:40:24.425Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"4bfdd7ab-97a1-4e5e-b6a6-c50477113096","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:40:24.425Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"4bfdd7ab-97a1-4e5e-b6a6-c50477113096","seconds":300}
{"level":"info","ts":"2026-06-05T09:40:26.993Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"36004e3c-fcea-47c9-a30f-f2edf42224e8"}
{"level":"info","ts":"2026-06-05T09:40:26.993Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"36004e3c-fcea-47c9-a30f-f2edf42224e8"}
{"level":"info","ts":"2026-06-05T09:40:26.993Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"36004e3c-fcea-47c9-a30f-f2edf42224e8","1 head service found":"stop-mmlfk-head-svc"}
{"level":"info","ts":"2026-06-05T09:40:26.993Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"36004e3c-fcea-47c9-a30f-f2edf42224e8","Found 1 head Pod":"stop-mmlfk-head-7pzr6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:40:26.993Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"36004e3c-fcea-47c9-a30f-f2edf42224e8","head Pod":"stop-mmlfk-head-7pzr6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-mmlfk-head-7pzr6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:40:26.993Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"36004e3c-fcea-47c9-a30f-f2edf42224e8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:40:26.993Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"36004e3c-fcea-47c9-a30f-f2edf42224e8","worker Pod":"stop-mmlfk-small-group-worker-26zt6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-mmlfk-small-group-worker-26zt6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:40:26.994Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"36004e3c-fcea-47c9-a30f-f2edf42224e8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:40:26.994Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"36004e3c-fcea-47c9-a30f-f2edf42224e8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:40:26.994Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"36004e3c-fcea-47c9-a30f-f2edf42224e8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:40:26.994Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"36004e3c-fcea-47c9-a30f-f2edf42224e8","podName":"stop-mmlfk-small-group-worker-26zt6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:40:26.994Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"36004e3c-fcea-47c9-a30f-f2edf42224e8","podName":"stop-mmlfk-small-group-worker-26zt6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:40:26.994Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"36004e3c-fcea-47c9-a30f-f2edf42224e8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:40:26.994Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"36004e3c-fcea-47c9-a30f-f2edf42224e8","seconds":300}
{"level":"info","ts":"2026-06-05T09:40:27.217Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"7d341bc7-6894-45dc-8c9e-ee6a6a8f4c64","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:40:27.217Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"7d341bc7-6894-45dc-8c9e-ee6a6a8f4c64","name":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:27.217Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"7d341bc7-6894-45dc-8c9e-ee6a6a8f4c64","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:40:27.217Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"7d341bc7-6894-45dc-8c9e-ee6a6a8f4c64"}
{"level":"info","ts":"2026-06-05T09:40:27.217Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"7d341bc7-6894-45dc-8c9e-ee6a6a8f4c64","RayCluster":"stop-mmlfk","State":""}
{"level":"info","ts":"2026-06-05T09:40:28.000Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"533d67f9-b646-44bd-b31f-64a85b086dce"}
{"level":"info","ts":"2026-06-05T09:40:28.000Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"533d67f9-b646-44bd-b31f-64a85b086dce"}
{"level":"info","ts":"2026-06-05T09:40:28.001Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"533d67f9-b646-44bd-b31f-64a85b086dce","1 head service found":"stop-mmlfk-head-svc"}
{"level":"info","ts":"2026-06-05T09:40:28.001Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"533d67f9-b646-44bd-b31f-64a85b086dce","Found 1 head Pod":"stop-mmlfk-head-7pzr6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:40:28.001Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"533d67f9-b646-44bd-b31f-64a85b086dce","head Pod":"stop-mmlfk-head-7pzr6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-mmlfk-head-7pzr6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:40:28.001Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"533d67f9-b646-44bd-b31f-64a85b086dce","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:40:28.001Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"533d67f9-b646-44bd-b31f-64a85b086dce","worker Pod":"stop-mmlfk-small-group-worker-26zt6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-mmlfk-small-group-worker-26zt6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:40:28.001Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"533d67f9-b646-44bd-b31f-64a85b086dce","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:40:28.001Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"533d67f9-b646-44bd-b31f-64a85b086dce","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:40:28.001Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"533d67f9-b646-44bd-b31f-64a85b086dce","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:40:28.001Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"533d67f9-b646-44bd-b31f-64a85b086dce","podName":"stop-mmlfk-small-group-worker-26zt6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:40:03Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-05T09:40:28.001Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"533d67f9-b646-44bd-b31f-64a85b086dce","podName":"stop-mmlfk-small-group-worker-26zt6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:40:03Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-05T09:40:28.001Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"533d67f9-b646-44bd-b31f-64a85b086dce","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-05T09:40:28.001Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"533d67f9-b646-44bd-b31f-64a85b086dce","name":"stop-mmlfk","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:40:22Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.37","serviceIP":"10.133.0.37","podName":"stop-mmlfk-head-7pzr6","serviceName":"stop-mmlfk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:40:22Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:40:28Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.37","serviceIP":"10.133.0.37","podName":"stop-mmlfk-head-7pzr6","serviceName":"stop-mmlfk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:40:22Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:40:28.015Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"8b44e38c-08e0-40b6-917d-b811768b790f","rayCluster":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:28.015Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"fa02615b-a627-4156-a62a-2577ec36894d","cluster":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:28.015Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"8b44e38c-08e0-40b6-917d-b811768b790f","pod":"stop-mmlfk-head-7pzr6","ip":"10.133.0.37"}
{"level":"info","ts":"2026-06-05T09:40:28.015Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"8b44e38c-08e0-40b6-917d-b811768b790f","pod":"stop-mmlfk-small-group-worker-26zt6","ip":"10.132.0.17"}
{"level":"info","ts":"2026-06-05T09:40:28.015Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"163948dc-65e2-4009-9b3a-1de8155c232a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:40:28.015Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"163948dc-65e2-4009-9b3a-1de8155c232a","name":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:28.015Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"163948dc-65e2-4009-9b3a-1de8155c232a","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:40:28.015Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"8b44e38c-08e0-40b6-917d-b811768b790f","rayCluster":"stop-mmlfk","podIPs":["10.133.0.37","10.132.0.17"]}
{"level":"info","ts":"2026-06-05T09:40:28.015Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"163948dc-65e2-4009-9b3a-1de8155c232a"}
{"level":"info","ts":"2026-06-05T09:40:28.015Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"163948dc-65e2-4009-9b3a-1de8155c232a","RayCluster":"stop-mmlfk","State":""}
{"level":"info","ts":"2026-06-05T09:40:28.028Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"fa02615b-a627-4156-a62a-2577ec36894d","name":"stop-mmlfk-head"}
{"level":"info","ts":"2026-06-05T09:40:28.037Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"fa02615b-a627-4156-a62a-2577ec36894d","name":"stop-mmlfk-workers"}
{"level":"info","ts":"2026-06-05T09:40:28.037Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"fa02615b-a627-4156-a62a-2577ec36894d","cluster":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:30.015Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"32fa349e-5119-4667-9aa3-3001c28b1075"}
{"level":"info","ts":"2026-06-05T09:40:30.015Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"32fa349e-5119-4667-9aa3-3001c28b1075"}
{"level":"info","ts":"2026-06-05T09:40:30.015Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"32fa349e-5119-4667-9aa3-3001c28b1075","1 head service found":"stop-mmlfk-head-svc"}
{"level":"info","ts":"2026-06-05T09:40:30.015Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"32fa349e-5119-4667-9aa3-3001c28b1075","Found 1 head Pod":"stop-mmlfk-head-7pzr6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:40:30.015Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"32fa349e-5119-4667-9aa3-3001c28b1075","head Pod":"stop-mmlfk-head-7pzr6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-mmlfk-head-7pzr6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:40:30.015Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"32fa349e-5119-4667-9aa3-3001c28b1075","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:40:30.015Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"32fa349e-5119-4667-9aa3-3001c28b1075","worker Pod":"stop-mmlfk-small-group-worker-26zt6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-mmlfk-small-group-worker-26zt6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:40:30.015Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"32fa349e-5119-4667-9aa3-3001c28b1075","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:40:30.015Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"32fa349e-5119-4667-9aa3-3001c28b1075","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:40:30.015Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"32fa349e-5119-4667-9aa3-3001c28b1075","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:40:30.015Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"32fa349e-5119-4667-9aa3-3001c28b1075","podName":"stop-mmlfk-small-group-worker-26zt6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:40:03Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-05T09:40:30.015Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"32fa349e-5119-4667-9aa3-3001c28b1075","podName":"stop-mmlfk-small-group-worker-26zt6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:40:03Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-05T09:40:30.015Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"32fa349e-5119-4667-9aa3-3001c28b1075","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:40:30.015Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"32fa349e-5119-4667-9aa3-3001c28b1075","seconds":300}
{"level":"info","ts":"2026-06-05T09:40:30.218Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"94b45100-d26b-4048-9bb3-9ea8a077529d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:40:30.218Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"94b45100-d26b-4048-9bb3-9ea8a077529d","name":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:30.218Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"94b45100-d26b-4048-9bb3-9ea8a077529d","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:40:30.218Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"94b45100-d26b-4048-9bb3-9ea8a077529d"}
{"level":"info","ts":"2026-06-05T09:40:30.218Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"94b45100-d26b-4048-9bb3-9ea8a077529d","RayCluster":"stop-mmlfk","State":""}
{"level":"info","ts":"2026-06-05T09:40:30.412Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"049e5d7c-cd22-42df-b060-083a97a8d566","rayCluster":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:30.412Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"049e5d7c-cd22-42df-b060-083a97a8d566","pod":"stop-mmlfk-head-7pzr6","ip":"10.133.0.37"}
{"level":"info","ts":"2026-06-05T09:40:30.412Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"049e5d7c-cd22-42df-b060-083a97a8d566","pod":"stop-mmlfk-small-group-worker-26zt6","ip":"10.132.0.17"}
{"level":"info","ts":"2026-06-05T09:40:30.413Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"049e5d7c-cd22-42df-b060-083a97a8d566","rayCluster":"stop-mmlfk","podIPs":["10.133.0.37","10.132.0.17"]}
{"level":"info","ts":"2026-06-05T09:40:33.219Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"b60b61e6-a605-4f7c-a04a-640a7f134219","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:40:33.219Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"b60b61e6-a605-4f7c-a04a-640a7f134219","name":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:33.219Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"b60b61e6-a605-4f7c-a04a-640a7f134219","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:40:33.219Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"b60b61e6-a605-4f7c-a04a-640a7f134219"}
{"level":"info","ts":"2026-06-05T09:40:33.219Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"b60b61e6-a605-4f7c-a04a-640a7f134219","RayCluster":"stop-mmlfk","State":""}
{"level":"info","ts":"2026-06-05T09:40:36.220Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"ba58646d-fa5d-474d-8e45-7540efcf743e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:40:36.220Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"ba58646d-fa5d-474d-8e45-7540efcf743e","name":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:36.220Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"ba58646d-fa5d-474d-8e45-7540efcf743e","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:40:36.220Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"ba58646d-fa5d-474d-8e45-7540efcf743e"}
{"level":"info","ts":"2026-06-05T09:40:36.220Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"ba58646d-fa5d-474d-8e45-7540efcf743e","RayCluster":"stop-mmlfk","State":""}
{"level":"info","ts":"2026-06-05T09:40:36.549Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"3e5e1314-3104-4a9b-869a-a6cad1c88da3","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:40:36.549Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"3e5e1314-3104-4a9b-869a-a6cad1c88da3","pod":"raycluster-head-vjd7d","ip":"10.134.0.20"}
{"level":"info","ts":"2026-06-05T09:40:36.549Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"3e5e1314-3104-4a9b-869a-a6cad1c88da3","pod":"raycluster-small-group-worker-j9pbw","ip":"10.133.0.35"}
{"level":"info","ts":"2026-06-05T09:40:36.549Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"3e5e1314-3104-4a9b-869a-a6cad1c88da3","rayCluster":"raycluster","podIPs":["10.134.0.20","10.133.0.35"]}
{"level":"info","ts":"2026-06-05T09:40:39.027Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"c38145e3-75db-421c-aa4b-ccd9b74365cf"}
{"level":"info","ts":"2026-06-05T09:40:39.027Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"c38145e3-75db-421c-aa4b-ccd9b74365cf"}
{"level":"info","ts":"2026-06-05T09:40:39.027Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"c38145e3-75db-421c-aa4b-ccd9b74365cf","1 head service found":"stop-mmlfk-head-svc"}
{"level":"info","ts":"2026-06-05T09:40:39.027Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"c38145e3-75db-421c-aa4b-ccd9b74365cf","Found 1 head Pod":"stop-mmlfk-head-7pzr6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:40:39.027Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"c38145e3-75db-421c-aa4b-ccd9b74365cf","head Pod":"stop-mmlfk-head-7pzr6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-mmlfk-head-7pzr6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:40:39.027Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"c38145e3-75db-421c-aa4b-ccd9b74365cf","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:40:39.027Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"c38145e3-75db-421c-aa4b-ccd9b74365cf","worker Pod":"stop-mmlfk-small-group-worker-26zt6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-mmlfk-small-group-worker-26zt6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:40:39.027Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"c38145e3-75db-421c-aa4b-ccd9b74365cf","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:40:39.027Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"c38145e3-75db-421c-aa4b-ccd9b74365cf","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:40:39.027Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"c38145e3-75db-421c-aa4b-ccd9b74365cf","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:40:39.027Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"c38145e3-75db-421c-aa4b-ccd9b74365cf","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-06-05T09:40:39.027Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"c38145e3-75db-421c-aa4b-ccd9b74365cf","name":"stop-mmlfk","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:40:28Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.37","serviceIP":"10.133.0.37","podName":"stop-mmlfk-head-7pzr6","serviceName":"stop-mmlfk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:40:22Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:40:39Z","stateTransitionTimes":{"ready":"2026-06-05T09:40:39Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.37","serviceIP":"10.133.0.37","podName":"stop-mmlfk-head-7pzr6","serviceName":"stop-mmlfk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:40:22Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:40:39Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:40:39.043Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"2616a6b4-396a-4d1b-9e55-d8d5adba15c7","rayCluster":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:39.043Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"f0aaa945-033e-4f1a-a013-c7de1ceb099d","cluster":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:39.043Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"1dcce1b4-1d57-4166-9d83-24e50f510da6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:40:39.043Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"1dcce1b4-1d57-4166-9d83-24e50f510da6","name":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:39.043Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"2616a6b4-396a-4d1b-9e55-d8d5adba15c7","pod":"stop-mmlfk-small-group-worker-26zt6","ip":"10.132.0.17"}
{"level":"info","ts":"2026-06-05T09:40:39.043Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"2616a6b4-396a-4d1b-9e55-d8d5adba15c7","pod":"stop-mmlfk-head-7pzr6","ip":"10.133.0.37"}
{"level":"info","ts":"2026-06-05T09:40:39.043Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"1dcce1b4-1d57-4166-9d83-24e50f510da6","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:40:39.043Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"2616a6b4-396a-4d1b-9e55-d8d5adba15c7","rayCluster":"stop-mmlfk","podIPs":["10.132.0.17","10.133.0.37"]}
{"level":"info","ts":"2026-06-05T09:40:39.043Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"1dcce1b4-1d57-4166-9d83-24e50f510da6"}
{"level":"info","ts":"2026-06-05T09:40:39.043Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"1dcce1b4-1d57-4166-9d83-24e50f510da6","head service name":"stop-mmlfk-head-svc","namespace":"test-ns-l4cdh"}
{"level":"info","ts":"2026-06-05T09:40:39.043Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"1dcce1b4-1d57-4166-9d83-24e50f510da6","head service URL":"stop-mmlfk-head-svc.test-ns-l4cdh.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-05T09:40:39.043Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"1dcce1b4-1d57-4166-9d83-24e50f510da6","SubmissionMode":"HTTPMode","RayCluster":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:39.043Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"1dcce1b4-1d57-4166-9d83-24e50f510da6","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-m4mp7","rayClusterName":"stop-mmlfk","jobDeploymentStatus":"Initializing","startTime":"2026-06-05T09:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-m4mp7","rayClusterName":"stop-mmlfk","dashboardURL":"stop-mmlfk-head-svc.test-ns-l4cdh.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-05T09:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:40:39.043Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"1dcce1b4-1d57-4166-9d83-24e50f510da6","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-05T09:40:39.054Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"f0aaa945-033e-4f1a-a013-c7de1ceb099d","name":"stop-mmlfk-head"}
{"level":"info","ts":"2026-06-05T09:40:39.054Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"d61a030b-1ff8-4156-be07-283c9222c1c9","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:40:39.054Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"d61a030b-1ff8-4156-be07-283c9222c1c9","name":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:39.054Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"d61a030b-1ff8-4156-be07-283c9222c1c9","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:40:39.054Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"d61a030b-1ff8-4156-be07-283c9222c1c9"}
{"level":"info","ts":"2026-06-05T09:40:39.063Z","logger":"controllers.RayJob","msg":"The Ray job was not found. Submit a Ray job via an HTTP request.","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"d61a030b-1ff8-4156-be07-283c9222c1c9","JobId":"stop-m4mp7"}
{"level":"info","ts":"2026-06-05T09:40:39.063Z","logger":"controllers.RayJob","msg":"Submit a ray job","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"d61a030b-1ff8-4156-be07-283c9222c1c9","rayJob":"stop","jobInfo":"{\"entrypoint\":\"python /home/ray/jobs/stop.py\",\"submission_id\":\"stop-m4mp7\"}"}
{"level":"info","ts":"2026-06-05T09:40:39.066Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"f0aaa945-033e-4f1a-a013-c7de1ceb099d","name":"stop-mmlfk-workers"}
{"level":"info","ts":"2026-06-05T09:40:39.066Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"f0aaa945-033e-4f1a-a013-c7de1ceb099d","cluster":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:39.959Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"feacf8c7-9627-4ba2-9450-3b21ac89bdd4","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:40:39.959Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"feacf8c7-9627-4ba2-9450-3b21ac89bdd4","name":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:39.959Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"feacf8c7-9627-4ba2-9450-3b21ac89bdd4","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:40:39.959Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"feacf8c7-9627-4ba2-9450-3b21ac89bdd4"}
{"level":"info","ts":"2026-06-05T09:40:39.962Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"feacf8c7-9627-4ba2-9450-3b21ac89bdd4","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-m4mp7","rayClusterName":"stop-mmlfk","dashboardURL":"stop-mmlfk-head-svc.test-ns-l4cdh.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-05T09:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:40:39Z"},"jobId":"stop-m4mp7","rayClusterName":"stop-mmlfk","dashboardURL":"stop-mmlfk-head-svc.test-ns-l4cdh.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-05T09:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:40:39Z","stateTransitionTimes":{"ready":"2026-06-05T09:40:39Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.37","serviceIP":"10.133.0.37","podName":"stop-mmlfk-head-7pzr6","serviceName":"stop-mmlfk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:40:22Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:40:39Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:40:39.962Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"feacf8c7-9627-4ba2-9450-3b21ac89bdd4","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-05T09:40:39.977Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"c7f91980-401d-4f55-8669-bf88f623592a","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:40:39.977Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"c7f91980-401d-4f55-8669-bf88f623592a","name":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:39.977Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"c7f91980-401d-4f55-8669-bf88f623592a","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:40:39.977Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"c7f91980-401d-4f55-8669-bf88f623592a"}
{"level":"info","ts":"2026-06-05T09:40:39.981Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"c7f91980-401d-4f55-8669-bf88f623592a","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:40:39Z"},"jobId":"stop-m4mp7","rayClusterName":"stop-mmlfk","dashboardURL":"stop-mmlfk-head-svc.test-ns-l4cdh.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-05T09:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:40:39Z","stateTransitionTimes":{"ready":"2026-06-05T09:40:39Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.37","serviceIP":"10.133.0.37","podName":"stop-mmlfk-head-7pzr6","serviceName":"stop-mmlfk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:40:22Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:40:39Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:40:39Z"},"jobId":"stop-m4mp7","rayClusterName":"stop-mmlfk","dashboardURL":"stop-mmlfk-head-svc.test-ns-l4cdh.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-05T09:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:40:39Z","stateTransitionTimes":{"ready":"2026-06-05T09:40:39Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.37","serviceIP":"10.133.0.37","podName":"stop-mmlfk-head-7pzr6","serviceName":"stop-mmlfk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:40:22Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:40:39Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:40:41.042Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"7da89558-c75c-46fa-9f80-95bd7fd88d21"}
{"level":"info","ts":"2026-06-05T09:40:41.042Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"7da89558-c75c-46fa-9f80-95bd7fd88d21"}
{"level":"info","ts":"2026-06-05T09:40:41.042Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"7da89558-c75c-46fa-9f80-95bd7fd88d21","1 head service found":"stop-mmlfk-head-svc"}
{"level":"info","ts":"2026-06-05T09:40:41.043Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"7da89558-c75c-46fa-9f80-95bd7fd88d21","Found 1 head Pod":"stop-mmlfk-head-7pzr6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:40:41.043Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"7da89558-c75c-46fa-9f80-95bd7fd88d21","head Pod":"stop-mmlfk-head-7pzr6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-mmlfk-head-7pzr6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:40:41.043Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"7da89558-c75c-46fa-9f80-95bd7fd88d21","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:40:41.043Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"7da89558-c75c-46fa-9f80-95bd7fd88d21","worker Pod":"stop-mmlfk-small-group-worker-26zt6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-mmlfk-small-group-worker-26zt6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:40:41.043Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"7da89558-c75c-46fa-9f80-95bd7fd88d21","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:40:41.043Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"7da89558-c75c-46fa-9f80-95bd7fd88d21","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:40:41.043Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"7da89558-c75c-46fa-9f80-95bd7fd88d21","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:40:41.043Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"7da89558-c75c-46fa-9f80-95bd7fd88d21","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:40:41.043Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"7da89558-c75c-46fa-9f80-95bd7fd88d21","seconds":300}
{"level":"info","ts":"2026-06-05T09:40:42.959Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"83063a94-2609-4e57-8bf6-1bfcb7d868db","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:40:42.959Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"83063a94-2609-4e57-8bf6-1bfcb7d868db","name":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:42.959Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"83063a94-2609-4e57-8bf6-1bfcb7d868db","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:40:42.959Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"83063a94-2609-4e57-8bf6-1bfcb7d868db"}
{"level":"info","ts":"2026-06-05T09:40:42.963Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"83063a94-2609-4e57-8bf6-1bfcb7d868db","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:40:39Z"},"jobId":"stop-m4mp7","rayClusterName":"stop-mmlfk","dashboardURL":"stop-mmlfk-head-svc.test-ns-l4cdh.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-05T09:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:40:39Z","stateTransitionTimes":{"ready":"2026-06-05T09:40:39Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.37","serviceIP":"10.133.0.37","podName":"stop-mmlfk-head-7pzr6","serviceName":"stop-mmlfk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:40:22Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:40:39Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:40:39Z"},"jobId":"stop-m4mp7","rayClusterName":"stop-mmlfk","dashboardURL":"stop-mmlfk-head-svc.test-ns-l4cdh.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-05T09:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:40:39Z","stateTransitionTimes":{"ready":"2026-06-05T09:40:39Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.37","serviceIP":"10.133.0.37","podName":"stop-mmlfk-head-7pzr6","serviceName":"stop-mmlfk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:40:22Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:40:39Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:40:42.963Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"83063a94-2609-4e57-8bf6-1bfcb7d868db","old JobStatus":"PENDING","new JobStatus":"RUNNING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-05T09:40:42.978Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"d973be00-4123-47ec-95f9-38cfecacf181","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:40:42.978Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"d973be00-4123-47ec-95f9-38cfecacf181","name":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:42.978Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"d973be00-4123-47ec-95f9-38cfecacf181","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:40:42.978Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"d973be00-4123-47ec-95f9-38cfecacf181"}
{"level":"info","ts":"2026-06-05T09:40:42.982Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"d973be00-4123-47ec-95f9-38cfecacf181","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:40:39Z"},"jobId":"stop-m4mp7","rayClusterName":"stop-mmlfk","dashboardURL":"stop-mmlfk-head-svc.test-ns-l4cdh.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-05T09:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:40:39Z","stateTransitionTimes":{"ready":"2026-06-05T09:40:39Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.37","serviceIP":"10.133.0.37","podName":"stop-mmlfk-head-7pzr6","serviceName":"stop-mmlfk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:40:22Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:40:39Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:40:39Z"},"jobId":"stop-m4mp7","rayClusterName":"stop-mmlfk","dashboardURL":"stop-mmlfk-head-svc.test-ns-l4cdh.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-05T09:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:40:39Z","stateTransitionTimes":{"ready":"2026-06-05T09:40:39Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.37","serviceIP":"10.133.0.37","podName":"stop-mmlfk-head-7pzr6","serviceName":"stop-mmlfk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:40:22Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:40:39Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:40:45.977Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"65026c6f-e024-4c4a-bb07-5d7e727237fd","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:40:45.977Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"65026c6f-e024-4c4a-bb07-5d7e727237fd","name":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:45.977Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"65026c6f-e024-4c4a-bb07-5d7e727237fd","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:40:45.977Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"65026c6f-e024-4c4a-bb07-5d7e727237fd"}
{"level":"info","ts":"2026-06-05T09:40:45.981Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"65026c6f-e024-4c4a-bb07-5d7e727237fd","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:40:39Z"},"jobId":"stop-m4mp7","rayClusterName":"stop-mmlfk","dashboardURL":"stop-mmlfk-head-svc.test-ns-l4cdh.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-05T09:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:40:39Z","stateTransitionTimes":{"ready":"2026-06-05T09:40:39Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.37","serviceIP":"10.133.0.37","podName":"stop-mmlfk-head-7pzr6","serviceName":"stop-mmlfk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:40:22Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:40:39Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:40:39Z"},"jobId":"stop-m4mp7","rayClusterName":"stop-mmlfk","dashboardURL":"stop-mmlfk-head-svc.test-ns-l4cdh.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-05T09:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:40:39Z","stateTransitionTimes":{"ready":"2026-06-05T09:40:39Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.37","serviceIP":"10.133.0.37","podName":"stop-mmlfk-head-7pzr6","serviceName":"stop-mmlfk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:40:22Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:40:39Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:40:48.982Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"cf03595b-abe1-4678-baa8-92ee987457f2","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:40:48.982Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"cf03595b-abe1-4678-baa8-92ee987457f2","name":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:48.982Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"cf03595b-abe1-4678-baa8-92ee987457f2","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:40:48.982Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"cf03595b-abe1-4678-baa8-92ee987457f2"}
{"level":"info","ts":"2026-06-05T09:40:48.985Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"cf03595b-abe1-4678-baa8-92ee987457f2","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:40:39Z"},"jobId":"stop-m4mp7","rayClusterName":"stop-mmlfk","dashboardURL":"stop-mmlfk-head-svc.test-ns-l4cdh.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-05T09:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:40:39Z","stateTransitionTimes":{"ready":"2026-06-05T09:40:39Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.37","serviceIP":"10.133.0.37","podName":"stop-mmlfk-head-7pzr6","serviceName":"stop-mmlfk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:40:22Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:40:39Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:40:39Z"},"jobId":"stop-m4mp7","rayClusterName":"stop-mmlfk","dashboardURL":"stop-mmlfk-head-svc.test-ns-l4cdh.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-05T09:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:40:39Z","stateTransitionTimes":{"ready":"2026-06-05T09:40:39Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.37","serviceIP":"10.133.0.37","podName":"stop-mmlfk-head-7pzr6","serviceName":"stop-mmlfk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:40:22Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:40:39Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:40:51.986Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"3cd66ac7-3055-4d11-8806-429c23ab4212","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:40:51.986Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"3cd66ac7-3055-4d11-8806-429c23ab4212","name":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:51.986Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"3cd66ac7-3055-4d11-8806-429c23ab4212","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:40:51.986Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"3cd66ac7-3055-4d11-8806-429c23ab4212"}
{"level":"info","ts":"2026-06-05T09:40:51.989Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"3cd66ac7-3055-4d11-8806-429c23ab4212","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:40:39Z"},"jobId":"stop-m4mp7","rayClusterName":"stop-mmlfk","dashboardURL":"stop-mmlfk-head-svc.test-ns-l4cdh.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-05T09:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:40:39Z","stateTransitionTimes":{"ready":"2026-06-05T09:40:39Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.37","serviceIP":"10.133.0.37","podName":"stop-mmlfk-head-7pzr6","serviceName":"stop-mmlfk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:40:22Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:40:39Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:40:39Z"},"jobId":"stop-m4mp7","rayClusterName":"stop-mmlfk","dashboardURL":"stop-mmlfk-head-svc.test-ns-l4cdh.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-05T09:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:40:39Z","stateTransitionTimes":{"ready":"2026-06-05T09:40:39Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.37","serviceIP":"10.133.0.37","podName":"stop-mmlfk-head-7pzr6","serviceName":"stop-mmlfk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:40:22Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:40:39Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:40:54.989Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"d81abf2c-4a56-4c10-bcc3-05314b095f63","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:40:54.989Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"d81abf2c-4a56-4c10-bcc3-05314b095f63","name":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:54.989Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"d81abf2c-4a56-4c10-bcc3-05314b095f63","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:40:54.989Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"d81abf2c-4a56-4c10-bcc3-05314b095f63"}
{"level":"info","ts":"2026-06-05T09:40:54.992Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"d81abf2c-4a56-4c10-bcc3-05314b095f63","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:40:39Z"},"jobId":"stop-m4mp7","rayClusterName":"stop-mmlfk","dashboardURL":"stop-mmlfk-head-svc.test-ns-l4cdh.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-05T09:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:40:39Z","stateTransitionTimes":{"ready":"2026-06-05T09:40:39Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.37","serviceIP":"10.133.0.37","podName":"stop-mmlfk-head-7pzr6","serviceName":"stop-mmlfk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:40:22Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:40:39Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:40:39Z"},"jobId":"stop-m4mp7","rayClusterName":"stop-mmlfk","dashboardURL":"stop-mmlfk-head-svc.test-ns-l4cdh.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-05T09:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:40:39Z","stateTransitionTimes":{"ready":"2026-06-05T09:40:39Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.37","serviceIP":"10.133.0.37","podName":"stop-mmlfk-head-7pzr6","serviceName":"stop-mmlfk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:40:22Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:40:39Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:40:57.993Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"d54f47f7-d9ea-4fcc-be3d-c041b76eeb77","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:40:57.993Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"d54f47f7-d9ea-4fcc-be3d-c041b76eeb77","name":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:40:57.993Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"d54f47f7-d9ea-4fcc-be3d-c041b76eeb77","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:40:57.993Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"d54f47f7-d9ea-4fcc-be3d-c041b76eeb77"}
{"level":"info","ts":"2026-06-05T09:40:57.996Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"d54f47f7-d9ea-4fcc-be3d-c041b76eeb77","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:40:39Z"},"jobId":"stop-m4mp7","rayClusterName":"stop-mmlfk","dashboardURL":"stop-mmlfk-head-svc.test-ns-l4cdh.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-05T09:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:40:39Z","stateTransitionTimes":{"ready":"2026-06-05T09:40:39Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.37","serviceIP":"10.133.0.37","podName":"stop-mmlfk-head-7pzr6","serviceName":"stop-mmlfk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:40:22Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:40:39Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:40:39Z"},"jobId":"stop-m4mp7","rayClusterName":"stop-mmlfk","dashboardURL":"stop-mmlfk-head-svc.test-ns-l4cdh.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-05T09:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:40:39Z","stateTransitionTimes":{"ready":"2026-06-05T09:40:39Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.37","serviceIP":"10.133.0.37","podName":"stop-mmlfk-head-7pzr6","serviceName":"stop-mmlfk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:40:22Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:40:39Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:41:00.407Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"96c88f5e-d7ac-4deb-99db-ab9b62e7a1cf","rayCluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:41:00.407Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"96c88f5e-d7ac-4deb-99db-ab9b62e7a1cf","pod":"fail-d7vcr-head-8br8s","ip":"10.132.0.16"}
{"level":"info","ts":"2026-06-05T09:41:00.407Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"96c88f5e-d7ac-4deb-99db-ab9b62e7a1cf","pod":"fail-d7vcr-small-group-worker-crtnw","ip":"10.133.0.36"}
{"level":"info","ts":"2026-06-05T09:41:00.407Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"96c88f5e-d7ac-4deb-99db-ab9b62e7a1cf","rayCluster":"fail-d7vcr","podIPs":["10.132.0.16","10.133.0.36"]}
{"level":"info","ts":"2026-06-05T09:41:00.997Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"8b6d9d8e-98c6-42e2-8198-443b7f6b5857","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:41:00.997Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"8b6d9d8e-98c6-42e2-8198-443b7f6b5857","name":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:41:00.997Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"8b6d9d8e-98c6-42e2-8198-443b7f6b5857","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:41:00.997Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"8b6d9d8e-98c6-42e2-8198-443b7f6b5857"}
{"level":"info","ts":"2026-06-05T09:41:01.000Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"8b6d9d8e-98c6-42e2-8198-443b7f6b5857","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:40:39Z"},"jobId":"stop-m4mp7","rayClusterName":"stop-mmlfk","dashboardURL":"stop-mmlfk-head-svc.test-ns-l4cdh.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-05T09:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:40:39Z","stateTransitionTimes":{"ready":"2026-06-05T09:40:39Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.37","serviceIP":"10.133.0.37","podName":"stop-mmlfk-head-7pzr6","serviceName":"stop-mmlfk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:40:22Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:40:39Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:40:39Z"},"jobId":"stop-m4mp7","rayClusterName":"stop-mmlfk","dashboardURL":"stop-mmlfk-head-svc.test-ns-l4cdh.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-05T09:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:40:39Z","stateTransitionTimes":{"ready":"2026-06-05T09:40:39Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.37","serviceIP":"10.133.0.37","podName":"stop-mmlfk-head-7pzr6","serviceName":"stop-mmlfk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:40:22Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:40:39Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:41:04.001Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"944e6226-1d93-40d5-942b-b5aec146d591","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:41:04.001Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"944e6226-1d93-40d5-942b-b5aec146d591","name":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:41:04.001Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"944e6226-1d93-40d5-942b-b5aec146d591","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:41:04.002Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"944e6226-1d93-40d5-942b-b5aec146d591"}
{"level":"info","ts":"2026-06-05T09:41:04.005Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"944e6226-1d93-40d5-942b-b5aec146d591","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:40:39Z"},"jobId":"stop-m4mp7","rayClusterName":"stop-mmlfk","dashboardURL":"stop-mmlfk-head-svc.test-ns-l4cdh.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-05T09:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:40:39Z","stateTransitionTimes":{"ready":"2026-06-05T09:40:39Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.37","serviceIP":"10.133.0.37","podName":"stop-mmlfk-head-7pzr6","serviceName":"stop-mmlfk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:40:22Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:40:39Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:40:39Z","endTime":"2026-06-05T09:41:02Z"},"jobId":"stop-m4mp7","rayClusterName":"stop-mmlfk","dashboardURL":"stop-mmlfk-head-svc.test-ns-l4cdh.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Complete","message":"Job was intentionally stopped.","startTime":"2026-06-05T09:40:00Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:40:39Z","stateTransitionTimes":{"ready":"2026-06-05T09:40:39Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.37","serviceIP":"10.133.0.37","podName":"stop-mmlfk-head-7pzr6","serviceName":"stop-mmlfk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:40:22Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:40:39Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:40:00Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:40:00Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:41:04.005Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"944e6226-1d93-40d5-942b-b5aec146d591","old JobStatus":"RUNNING","new JobStatus":"STOPPED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Complete"}
{"level":"info","ts":"2026-06-05T09:41:04.020Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"8c429ec5-e990-4c2e-b7da-a14c4459c907","JobStatus":"STOPPED","JobDeploymentStatus":"Complete","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:41:04.020Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"8c429ec5-e990-4c2e-b7da-a14c4459c907","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-05 09:41:04 +0000 UTC","Now":"2026-06-05T09:41:04.020Z","ShutdownTime":"2026-06-05T09:41:04.000Z"}
{"level":"info","ts":"2026-06-05T09:41:04.498Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"daf8d8ab-fba2-4787-8c78-2e7ccfc2f509","DeletionTimestamp":"2026-06-05 09:41:04 +0000 UTC"}
{"level":"info","ts":"2026-06-05T09:41:04.498Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"daf8d8ab-fba2-4787-8c78-2e7ccfc2f509","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-05T09:41:04.513Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"47fd0792-7a23-42ef-8c8a-b3d2ac6d666c"}
{"level":"info","ts":"2026-06-05T09:41:04.540Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"8aa6a60c-5fb5-405c-8ec0-e6b069e64fc3","namespacedName":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:41:04.540Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"5f930638-1ac3-4cdb-998d-89a6f3232f5e"}
{"level":"info","ts":"2026-06-05T09:41:04.540Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"8aa6a60c-5fb5-405c-8ec0-e6b069e64fc3","cluster":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:41:04.540Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"8aa6a60c-5fb5-405c-8ec0-e6b069e64fc3","cluster":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:41:04.540Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"reconcileID":"436a9941-5a14-49cc-8d3a-05b24cf13b1a"}
{"level":"info","ts":"2026-06-05T09:41:04.540Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"309e824e-7da3-49d2-b7a7-67883a8b7b10","rayCluster":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:41:04.541Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"050252f5-cca6-452a-8d91-390c7e34b655"}
{"level":"info","ts":"2026-06-05T09:41:04.541Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"8aa6a60c-5fb5-405c-8ec0-e6b069e64fc3","namespace":"test-ns-l4cdh","otherAuthClusters":1,"action":"will retain"}
{"level":"info","ts":"2026-06-05T09:41:04.541Z","logger":"authentication-controller","msg":"ReferenceGrant retained (other clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"8aa6a60c-5fb5-405c-8ec0-e6b069e64fc3","name":"kuberay-gateway-access","namespace":"test-ns-l4cdh","remainingClusters":1}
{"level":"info","ts":"2026-06-05T09:41:04.541Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"309e824e-7da3-49d2-b7a7-67883a8b7b10","pod":"stop-mmlfk-small-group-worker-26zt6","ip":"10.132.0.17"}
{"level":"info","ts":"2026-06-05T09:41:04.541Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"309e824e-7da3-49d2-b7a7-67883a8b7b10","pod":"stop-mmlfk-head-7pzr6","ip":"10.133.0.37"}
{"level":"info","ts":"2026-06-05T09:41:04.541Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"309e824e-7da3-49d2-b7a7-67883a8b7b10","rayCluster":"stop-mmlfk","podIPs":["10.132.0.17","10.133.0.37"]}
{"level":"info","ts":"2026-06-05T09:41:04.547Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"8aa6a60c-5fb5-405c-8ec0-e6b069e64fc3","name":"test-ns-l4cdh-stop-mmlfk","namespace":"opendatahub"}
{"level":"info","ts":"2026-06-05T09:41:04.552Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"8aa6a60c-5fb5-405c-8ec0-e6b069e64fc3","configMap":"kube-rbac-proxy-config-stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:41:04.557Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"8aa6a60c-5fb5-405c-8ec0-e6b069e64fc3","serviceAccount":"stop-mmlfk-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:41:04.564Z","logger":"raycluster-resource","msg":"default","name":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:41:04.564Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"stop-mmlfk","namespace":"test-ns-l4cdh"}
{"level":"info","ts":"2026-06-05T09:41:04.575Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"8aa6a60c-5fb5-405c-8ec0-e6b069e64fc3","cluster":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:41:04.575Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"4759d2fe-1fb6-47c8-8127-23147bab9f77","namespacedName":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:41:04.575Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"4759d2fe-1fb6-47c8-8127-23147bab9f77"}
{"level":"info","ts":"2026-06-05T09:41:04.575Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"86fe227f-54af-4e60-8a76-91e4d0a14dc5","namespace":"test-ns-l4cdh","clusterName":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:41:04.575Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"5ad49552-e962-485b-a3bf-6ac1a39d523c"}
{"level":"info","ts":"2026-06-05T09:41:04.575Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"4759d2fe-1fb6-47c8-8127-23147bab9f77","namespace":"test-ns-l4cdh","authClusters":1}
{"level":"info","ts":"2026-06-05T09:41:04.575Z","logger":"authentication-controller","msg":"ReferenceGrant not orphaned (clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"4759d2fe-1fb6-47c8-8127-23147bab9f77","namespace":"test-ns-l4cdh","authClusters":1}
{"level":"info","ts":"2026-06-05T09:41:04.575Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"95e8ab64-707f-4371-a450-ff7f13a80cd3","namespacedName":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:41:04.575Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"95e8ab64-707f-4371-a450-ff7f13a80cd3"}
{"level":"info","ts":"2026-06-05T09:41:04.575Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"ff8fe008-72c2-416d-a7ae-404ed619142a"}
{"level":"info","ts":"2026-06-05T09:41:04.575Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"95e8ab64-707f-4371-a450-ff7f13a80cd3","namespace":"test-ns-l4cdh","authClusters":1}
{"level":"info","ts":"2026-06-05T09:41:04.575Z","logger":"authentication-controller","msg":"ReferenceGrant not orphaned (clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"95e8ab64-707f-4371-a450-ff7f13a80cd3","namespace":"test-ns-l4cdh","authClusters":1}
{"level":"info","ts":"2026-06-05T09:41:04.584Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"f1d1c8b7-e3f6-46b3-9ad2-9ed16749f881"}
{"level":"info","ts":"2026-06-05T09:41:04.606Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"e21314d3-7079-4b37-b159-5c1eb829d2ec","namespacedName":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:41:04.606Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"e21314d3-7079-4b37-b159-5c1eb829d2ec"}
{"level":"info","ts":"2026-06-05T09:41:04.606Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"a170efc4-19c6-455e-88b1-e183a5ebd5ee"}
{"level":"info","ts":"2026-06-05T09:41:04.606Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"e21314d3-7079-4b37-b159-5c1eb829d2ec","namespace":"test-ns-l4cdh","authClusters":1}
{"level":"info","ts":"2026-06-05T09:41:04.606Z","logger":"authentication-controller","msg":"ReferenceGrant not orphaned (clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"e21314d3-7079-4b37-b159-5c1eb829d2ec","namespace":"test-ns-l4cdh","authClusters":1}
{"level":"error","ts":"2026-06-05T09:41:04.610Z","msg":"Failed to delete head certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"86fe227f-54af-4e60-8a76-91e4d0a14dc5","certificate":"ray-head-cert-stop-mmlfk","error":"certificates.cert-manager.io \"ray-head-cert-stop-mmlfk\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:261\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:04.639Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"86fe227f-54af-4e60-8a76-91e4d0a14dc5","errorCount":1}
{"level":"info","ts":"2026-06-05T09:41:04.640Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"86fe227f-54af-4e60-8a76-91e4d0a14dc5"}
{"level":"error","ts":"2026-06-05T09:41:04.640Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"86fe227f-54af-4e60-8a76-91e4d0a14dc5","error":"certificates.cert-manager.io \"ray-head-cert-stop-mmlfk\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:04.645Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"9c512eaf-85bc-430e-967a-c1efc09fb5bd","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:04.645Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"9c512eaf-85bc-430e-967a-c1efc09fb5bd","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-05T09:41:04.645Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"ec6de2b5-002c-4bce-a143-eeddff277374","namespace":"test-ns-l4cdh","clusterName":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:41:04.645Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-mmlfk","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"stop-mmlfk","reconcileID":"ec6de2b5-002c-4bce-a143-eeddff277374","namespace":"test-ns-l4cdh","clusterName":"stop-mmlfk"}
{"level":"info","ts":"2026-06-05T09:41:04.656Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"9c512eaf-85bc-430e-967a-c1efc09fb5bd"}
{"level":"info","ts":"2026-06-05T09:41:04.656Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"9c512eaf-85bc-430e-967a-c1efc09fb5bd","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-05T09:41:04.656Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"9c512eaf-85bc-430e-967a-c1efc09fb5bd","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-wtb66","rayClusterName":"long-running-qkmps","jobDeploymentStatus":"Initializing","startTime":"2026-06-05T09:41:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:41:04.656Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"9c512eaf-85bc-430e-967a-c1efc09fb5bd","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-05T09:41:04.706Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"787536f7-c7bd-4f36-aa05-aa6cbf440a17","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:04.706Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"787536f7-c7bd-4f36-aa05-aa6cbf440a17"}
{"level":"info","ts":"2026-06-05T09:41:04.706Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"787536f7-c7bd-4f36-aa05-aa6cbf440a17","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-05T09:41:04.706Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"787536f7-c7bd-4f36-aa05-aa6cbf440a17","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-jfswv","rayClusterName":"long-running-smtnh","jobDeploymentStatus":"Initializing","startTime":"2026-06-05T09:41:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:41:04.706Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"787536f7-c7bd-4f36-aa05-aa6cbf440a17","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-05T09:41:04.714Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"787536f7-c7bd-4f36-aa05-aa6cbf440a17","error":"Operation cannot be fulfilled on rayjobs.ray.io \"long-running\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-06-05T09:41:04.714Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"787536f7-c7bd-4f36-aa05-aa6cbf440a17"}
{"level":"error","ts":"2026-06-05T09:41:04.714Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"787536f7-c7bd-4f36-aa05-aa6cbf440a17","error":"Operation cannot be fulfilled on rayjobs.ray.io \"long-running\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:04.715Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"aa8b9079-0c4b-4ecf-8278-2c168bc82676","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:04.715Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"aa8b9079-0c4b-4ecf-8278-2c168bc82676","name":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:04.715Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"aa8b9079-0c4b-4ecf-8278-2c168bc82676","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:04.715Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"aa8b9079-0c4b-4ecf-8278-2c168bc82676","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:04.719Z","logger":"raycluster-resource","msg":"default","name":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:04.719Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-qkmps","namespace":"test-ns-t5s4r"}
{"level":"info","ts":"2026-06-05T09:41:04.733Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"26d030e1-4226-4f85-bfba-f07edb544a4a","namespacedName":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:04.733Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"3d314477-3adf-4a1b-81a3-c0aafe4c8f83"}
{"level":"info","ts":"2026-06-05T09:41:04.733Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"3d314477-3adf-4a1b-81a3-c0aafe4c8f83"}
{"level":"info","ts":"2026-06-05T09:41:04.733Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"af77e38c-4734-4ec2-91bd-452ea4d71ed6","cluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:04.733Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"26d030e1-4226-4f85-bfba-f07edb544a4a","mode":"IntegratedOAuth","cluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:04.733Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"26d030e1-4226-4f85-bfba-f07edb544a4a","cluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:04.733Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"26d030e1-4226-4f85-bfba-f07edb544a4a","cluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:04.733Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"aa8b9079-0c4b-4ecf-8278-2c168bc82676","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:04.733Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"d0815000-94de-4309-83e2-3c5b7015d569","rayCluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:04.733Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"d0815000-94de-4309-83e2-3c5b7015d569","rayCluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:04.733Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"aa8b9079-0c4b-4ecf-8278-2c168bc82676"}
{"level":"info","ts":"2026-06-05T09:41:04.733Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"aa8b9079-0c4b-4ecf-8278-2c168bc82676","RayCluster":"long-running-qkmps","State":""}
{"level":"info","ts":"2026-06-05T09:41:04.733Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"3aab7ab0-2230-49df-8bb2-d5490a54b4c7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:04.733Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"3aab7ab0-2230-49df-8bb2-d5490a54b4c7","name":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:04.733Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"3aab7ab0-2230-49df-8bb2-d5490a54b4c7","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:04.733Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"3aab7ab0-2230-49df-8bb2-d5490a54b4c7"}
{"level":"info","ts":"2026-06-05T09:41:04.733Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"3aab7ab0-2230-49df-8bb2-d5490a54b4c7","RayCluster":"long-running-qkmps","State":""}
{"level":"info","ts":"2026-06-05T09:41:04.739Z","logger":"raycluster-resource","msg":"default","name":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:04.739Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-qkmps","namespace":"test-ns-t5s4r"}
{"level":"info","ts":"2026-06-05T09:41:04.806Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"3d314477-3adf-4a1b-81a3-c0aafe4c8f83","name":"long-running-qkmps-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:04.806Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"3d314477-3adf-4a1b-81a3-c0aafe4c8f83","error":"head certificate secret ray-head-secret-long-running-qkmps not found"}
{"level":"error","ts":"2026-06-05T09:41:04.806Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"3d314477-3adf-4a1b-81a3-c0aafe4c8f83","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qkmps not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:04.806Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"3d314477-3adf-4a1b-81a3-c0aafe4c8f83","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qkmps not found"}
{"level":"info","ts":"2026-06-05T09:41:04.806Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"3d314477-3adf-4a1b-81a3-c0aafe4c8f83","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-qkmps","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:04.806Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"3d314477-3adf-4a1b-81a3-c0aafe4c8f83","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-qkmps","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:04.806Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"3d314477-3adf-4a1b-81a3-c0aafe4c8f83","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-qkmps","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:04.806Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"3d314477-3adf-4a1b-81a3-c0aafe4c8f83","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-05T09:41:04.806Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"3d314477-3adf-4a1b-81a3-c0aafe4c8f83","name":"long-running-qkmps","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:41:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-qkmps-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:41:04.807Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"217d79ee-b721-4290-bfdc-52b0478b0cbc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:04.807Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"217d79ee-b721-4290-bfdc-52b0478b0cbc","name":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:04.807Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"217d79ee-b721-4290-bfdc-52b0478b0cbc","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:04.807Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"217d79ee-b721-4290-bfdc-52b0478b0cbc"}
{"level":"info","ts":"2026-06-05T09:41:04.807Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"217d79ee-b721-4290-bfdc-52b0478b0cbc","RayCluster":"long-running-qkmps","State":""}
{"level":"info","ts":"2026-06-05T09:41:04.814Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"26d030e1-4226-4f85-bfba-f07edb544a4a","name":"long-running-qkmps-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:41:04.815Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"3d314477-3adf-4a1b-81a3-c0aafe4c8f83","name":"long-running-qkmps","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-qkmps\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-t5s4r","name":"long-running-qkmps"}}
{"level":"info","ts":"2026-06-05T09:41:04.815Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"3d314477-3adf-4a1b-81a3-c0aafe4c8f83"}
{"level":"error","ts":"2026-06-05T09:41:04.815Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"3d314477-3adf-4a1b-81a3-c0aafe4c8f83","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qkmps not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:04.815Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"af77e38c-4734-4ec2-91bd-452ea4d71ed6","cluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:04.815Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"e0968bbb-4864-4579-bb7e-2e98b8b9d9db","cluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:04.815Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"95d64836-b34b-476d-b78e-14e96aaf174d"}
{"level":"info","ts":"2026-06-05T09:41:04.815Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"95d64836-b34b-476d-b78e-14e96aaf174d"}
{"level":"info","ts":"2026-06-05T09:41:04.815Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"95d64836-b34b-476d-b78e-14e96aaf174d","1 head service found":"long-running-qkmps-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:04.815Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"95d64836-b34b-476d-b78e-14e96aaf174d","error":"head certificate secret ray-head-secret-long-running-qkmps not found"}
{"level":"error","ts":"2026-06-05T09:41:04.815Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"95d64836-b34b-476d-b78e-14e96aaf174d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qkmps not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:04.815Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"95d64836-b34b-476d-b78e-14e96aaf174d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qkmps not found"}
{"level":"info","ts":"2026-06-05T09:41:04.816Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"95d64836-b34b-476d-b78e-14e96aaf174d","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-qkmps","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:04.816Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"95d64836-b34b-476d-b78e-14e96aaf174d","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-qkmps","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:04.816Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"95d64836-b34b-476d-b78e-14e96aaf174d","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-qkmps","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:04.816Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"95d64836-b34b-476d-b78e-14e96aaf174d","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-05T09:41:04.816Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"95d64836-b34b-476d-b78e-14e96aaf174d","name":"long-running-qkmps","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:41:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-qkmps-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:41:04.817Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"d0815000-94de-4309-83e2-3c5b7015d569","rayCluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:04.817Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"d0815000-94de-4309-83e2-3c5b7015d569","rayCluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:04.822Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"26d030e1-4226-4f85-bfba-f07edb544a4a","name":"kuberay-gateway-access","namespace":"test-ns-t5s4r","operation":"created"}
{"level":"info","ts":"2026-06-05T09:41:04.827Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"95d64836-b34b-476d-b78e-14e96aaf174d"}
{"level":"error","ts":"2026-06-05T09:41:04.827Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"95d64836-b34b-476d-b78e-14e96aaf174d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qkmps not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:04.827Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"8c1cda0e-bea8-419e-9836-d7b0691d3f31"}
{"level":"info","ts":"2026-06-05T09:41:04.827Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"8c1cda0e-bea8-419e-9836-d7b0691d3f31"}
{"level":"info","ts":"2026-06-05T09:41:04.827Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"d73f7816-041b-477f-96e1-16ef6fc9fb92","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:04.827Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"d73f7816-041b-477f-96e1-16ef6fc9fb92","name":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:04.827Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"d73f7816-041b-477f-96e1-16ef6fc9fb92","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:04.827Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"8c1cda0e-bea8-419e-9836-d7b0691d3f31","1 head service found":"long-running-qkmps-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:04.829Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"e0968bbb-4864-4579-bb7e-2e98b8b9d9db","name":"long-running-qkmps-head"}
{"level":"info","ts":"2026-06-05T09:41:04.905Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"8c1cda0e-bea8-419e-9836-d7b0691d3f31","error":"head certificate secret ray-head-secret-long-running-qkmps not found"}
{"level":"error","ts":"2026-06-05T09:41:04.905Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"8c1cda0e-bea8-419e-9836-d7b0691d3f31","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qkmps not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:04.906Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"8c1cda0e-bea8-419e-9836-d7b0691d3f31","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qkmps not found"}
{"level":"info","ts":"2026-06-05T09:41:04.906Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"d73f7816-041b-477f-96e1-16ef6fc9fb92"}
{"level":"info","ts":"2026-06-05T09:41:04.906Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"d73f7816-041b-477f-96e1-16ef6fc9fb92","RayCluster":"long-running-qkmps","State":""}
{"level":"info","ts":"2026-06-05T09:41:04.906Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"8c1cda0e-bea8-419e-9836-d7b0691d3f31","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-qkmps","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:04.906Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"8c1cda0e-bea8-419e-9836-d7b0691d3f31","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-qkmps","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:04.906Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"8c1cda0e-bea8-419e-9836-d7b0691d3f31","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-qkmps","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:04.906Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"26d030e1-4226-4f85-bfba-f07edb544a4a","name":"test-ns-t5s4r-long-running-qkmps","operation":"created"}
{"level":"info","ts":"2026-06-05T09:41:04.906Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"8c1cda0e-bea8-419e-9836-d7b0691d3f31"}
{"level":"info","ts":"2026-06-05T09:41:04.905Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"d0815000-94de-4309-83e2-3c5b7015d569"}
{"level":"error","ts":"2026-06-05T09:41:04.906Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"8c1cda0e-bea8-419e-9836-d7b0691d3f31","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qkmps not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:04.906Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"484eeba7-7668-47ce-98f2-1e81f6ef4c68"}
{"level":"info","ts":"2026-06-05T09:41:04.906Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"484eeba7-7668-47ce-98f2-1e81f6ef4c68"}
{"level":"info","ts":"2026-06-05T09:41:04.906Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"484eeba7-7668-47ce-98f2-1e81f6ef4c68","1 head service found":"long-running-qkmps-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:04.906Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"484eeba7-7668-47ce-98f2-1e81f6ef4c68","error":"head certificate secret ray-head-secret-long-running-qkmps not found"}
{"level":"info","ts":"2026-06-05T09:41:04.906Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"d0815000-94de-4309-83e2-3c5b7015d569","rayCluster":"long-running-qkmps","secret":"ca-secret"}
{"level":"info","ts":"2026-06-05T09:41:04.906Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"d0815000-94de-4309-83e2-3c5b7015d569","rayCluster":"long-running-qkmps"}
{"level":"error","ts":"2026-06-05T09:41:04.906Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"484eeba7-7668-47ce-98f2-1e81f6ef4c68","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qkmps not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:04.906Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"484eeba7-7668-47ce-98f2-1e81f6ef4c68","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qkmps not found"}
{"level":"info","ts":"2026-06-05T09:41:04.911Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"e0968bbb-4864-4579-bb7e-2e98b8b9d9db","name":"long-running-qkmps-workers"}
{"level":"info","ts":"2026-06-05T09:41:04.911Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"e0968bbb-4864-4579-bb7e-2e98b8b9d9db","cluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:04.911Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"049f12e7-9059-4ea0-b47f-61b723305bf6","cluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:04.912Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"26d030e1-4226-4f85-bfba-f07edb544a4a","name":"kube-rbac-proxy-config-long-running-qkmps","operation":"created"}
{"level":"info","ts":"2026-06-05T09:41:04.918Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"d0815000-94de-4309-83e2-3c5b7015d569","rayCluster":"long-running-qkmps"}
{"level":"error","ts":"2026-06-05T09:41:04.918Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"26d030e1-4226-4f85-bfba-f07edb544a4a","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-qkmps\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:04.919Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"26d030e1-4226-4f85-bfba-f07edb544a4a"}
{"level":"error","ts":"2026-06-05T09:41:04.919Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"26d030e1-4226-4f85-bfba-f07edb544a4a","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-qkmps\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:04.919Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"524a1dce-f21c-49f5-8502-36d9949e97ea","namespacedName":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:04.919Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"524a1dce-f21c-49f5-8502-36d9949e97ea","mode":"IntegratedOAuth","cluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:04.919Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"524a1dce-f21c-49f5-8502-36d9949e97ea","cluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:04.926Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"049f12e7-9059-4ea0-b47f-61b723305bf6","name":"long-running-qkmps-head"}
{"level":"info","ts":"2026-06-05T09:41:04.927Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"d0815000-94de-4309-83e2-3c5b7015d569"}
{"level":"info","ts":"2026-06-05T09:41:04.927Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"d0815000-94de-4309-83e2-3c5b7015d569","rayCluster":"long-running-qkmps","podIPs":[]}
{"level":"info","ts":"2026-06-05T09:41:05.005Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"484eeba7-7668-47ce-98f2-1e81f6ef4c68","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-qkmps","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:05.006Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"484eeba7-7668-47ce-98f2-1e81f6ef4c68","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-qkmps","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:05.006Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"484eeba7-7668-47ce-98f2-1e81f6ef4c68","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-qkmps","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:05.006Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"484eeba7-7668-47ce-98f2-1e81f6ef4c68"}
{"level":"error","ts":"2026-06-05T09:41:05.006Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"484eeba7-7668-47ce-98f2-1e81f6ef4c68","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qkmps not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:05.006Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"23945be7-27c6-4450-82c5-1c54a8ad33e5"}
{"level":"info","ts":"2026-06-05T09:41:05.006Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"23945be7-27c6-4450-82c5-1c54a8ad33e5"}
{"level":"info","ts":"2026-06-05T09:41:05.006Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"23945be7-27c6-4450-82c5-1c54a8ad33e5","1 head service found":"long-running-qkmps-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:05.006Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"23945be7-27c6-4450-82c5-1c54a8ad33e5","error":"head certificate secret ray-head-secret-long-running-qkmps not found"}
{"level":"error","ts":"2026-06-05T09:41:05.006Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"23945be7-27c6-4450-82c5-1c54a8ad33e5","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qkmps not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:05.006Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"23945be7-27c6-4450-82c5-1c54a8ad33e5","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qkmps not found"}
{"level":"info","ts":"2026-06-05T09:41:05.007Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"23945be7-27c6-4450-82c5-1c54a8ad33e5","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-qkmps","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:05.007Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"23945be7-27c6-4450-82c5-1c54a8ad33e5","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-qkmps","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:05.007Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"23945be7-27c6-4450-82c5-1c54a8ad33e5","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-qkmps","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:05.007Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"23945be7-27c6-4450-82c5-1c54a8ad33e5"}
{"level":"error","ts":"2026-06-05T09:41:05.007Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"23945be7-27c6-4450-82c5-1c54a8ad33e5","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qkmps not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:05.007Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"524a1dce-f21c-49f5-8502-36d9949e97ea","cluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:05.007Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"23cfbf54-5c94-4f40-a7fe-d026a334b260","namespacedName":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:05.007Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"23cfbf54-5c94-4f40-a7fe-d026a334b260","mode":"IntegratedOAuth","cluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:05.007Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"23cfbf54-5c94-4f40-a7fe-d026a334b260","cluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:05.009Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"735307fa-aee1-49da-8798-a950b2a2e143","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:05.009Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"735307fa-aee1-49da-8798-a950b2a2e143","name":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:05.009Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"735307fa-aee1-49da-8798-a950b2a2e143","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:05.009Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"735307fa-aee1-49da-8798-a950b2a2e143"}
{"level":"info","ts":"2026-06-05T09:41:05.009Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"735307fa-aee1-49da-8798-a950b2a2e143","RayCluster":"long-running-qkmps","State":""}
{"level":"info","ts":"2026-06-05T09:41:05.009Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"d0815000-94de-4309-83e2-3c5b7015d569"}
{"level":"info","ts":"2026-06-05T09:41:05.009Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"d0815000-94de-4309-83e2-3c5b7015d569","rayCluster":"long-running-qkmps","podIPs":[]}
{"level":"error","ts":"2026-06-05T09:41:05.009Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"d0815000-94de-4309-83e2-3c5b7015d569","error":"Certificate.cert-manager.io \"ray-worker-cert-long-running-qkmps\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:05.009Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"d0815000-94de-4309-83e2-3c5b7015d569"}
{"level":"error","ts":"2026-06-05T09:41:05.010Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"d0815000-94de-4309-83e2-3c5b7015d569","error":"Certificate.cert-manager.io \"ray-worker-cert-long-running-qkmps\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:05.010Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"bf05d1ab-d41d-4291-a8a5-af9702041f2c","rayCluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:05.106Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"3854dfde-0a38-4acd-9fb1-ef826206a980"}
{"level":"info","ts":"2026-06-05T09:41:05.106Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"3854dfde-0a38-4acd-9fb1-ef826206a980"}
{"level":"info","ts":"2026-06-05T09:41:05.106Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"3854dfde-0a38-4acd-9fb1-ef826206a980","1 head service found":"long-running-qkmps-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:05.106Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"3854dfde-0a38-4acd-9fb1-ef826206a980","error":"head certificate secret ray-head-secret-long-running-qkmps not found"}
{"level":"error","ts":"2026-06-05T09:41:05.106Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"3854dfde-0a38-4acd-9fb1-ef826206a980","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qkmps not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:05.106Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"bf05d1ab-d41d-4291-a8a5-af9702041f2c"}
{"level":"info","ts":"2026-06-05T09:41:05.106Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"3854dfde-0a38-4acd-9fb1-ef826206a980","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qkmps not found"}
{"level":"info","ts":"2026-06-05T09:41:05.106Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"049f12e7-9059-4ea0-b47f-61b723305bf6","name":"long-running-qkmps-workers"}
{"level":"info","ts":"2026-06-05T09:41:05.106Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"049f12e7-9059-4ea0-b47f-61b723305bf6","cluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:05.106Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"2a115dd6-86db-4e0c-badf-914bdd77f749","cluster":"long-running-qkmps"}
{"level":"error","ts":"2026-06-05T09:41:05.106Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"bf05d1ab-d41d-4291-a8a5-af9702041f2c","error":"certificates.cert-manager.io \"ray-worker-cert-long-running-qkmps\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createRayWorkerCertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:732\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:206\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-05T09:41:05.106Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"bf05d1ab-d41d-4291-a8a5-af9702041f2c","error":"certificates.cert-manager.io \"ray-worker-cert-long-running-qkmps\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:207\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:05.106Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"bf05d1ab-d41d-4291-a8a5-af9702041f2c"}
{"level":"error","ts":"2026-06-05T09:41:05.106Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"bf05d1ab-d41d-4291-a8a5-af9702041f2c","error":"certificates.cert-manager.io \"ray-worker-cert-long-running-qkmps\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:05.106Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"4e6c1468-30c7-41e2-ac8c-df77d96eab6f","rayCluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:05.106Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"4e6c1468-30c7-41e2-ac8c-df77d96eab6f"}
{"level":"info","ts":"2026-06-05T09:41:05.106Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"3854dfde-0a38-4acd-9fb1-ef826206a980","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-qkmps","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:05.107Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"3854dfde-0a38-4acd-9fb1-ef826206a980","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-qkmps","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:05.107Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"3854dfde-0a38-4acd-9fb1-ef826206a980","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-qkmps","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:05.107Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"3854dfde-0a38-4acd-9fb1-ef826206a980"}
{"level":"error","ts":"2026-06-05T09:41:05.107Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"3854dfde-0a38-4acd-9fb1-ef826206a980","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qkmps not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-05T09:41:05.106Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"23cfbf54-5c94-4f40-a7fe-d026a334b260","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-qkmps\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:05.107Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"23cfbf54-5c94-4f40-a7fe-d026a334b260"}
{"level":"error","ts":"2026-06-05T09:41:05.107Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"23cfbf54-5c94-4f40-a7fe-d026a334b260","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-qkmps\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:05.112Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"38631812-b071-44ed-82c7-57852400d857","namespacedName":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:05.112Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"38631812-b071-44ed-82c7-57852400d857","mode":"IntegratedOAuth","cluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:05.112Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"38631812-b071-44ed-82c7-57852400d857","cluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:05.116Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"4478f29f-7bff-4e5e-bbae-c3d3c0745661","rayCluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:05.118Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"2a115dd6-86db-4e0c-badf-914bdd77f749","name":"long-running-qkmps-head"}
{"level":"info","ts":"2026-06-05T09:41:05.120Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"4478f29f-7bff-4e5e-bbae-c3d3c0745661"}
{"level":"info","ts":"2026-06-05T09:41:05.124Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"38631812-b071-44ed-82c7-57852400d857","cluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:05.136Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"2a115dd6-86db-4e0c-badf-914bdd77f749","name":"long-running-qkmps-workers"}
{"level":"info","ts":"2026-06-05T09:41:05.136Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"2a115dd6-86db-4e0c-badf-914bdd77f749","cluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:05.267Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"7265fae5-1c6d-4609-bed7-2155c09c2aa3"}
{"level":"info","ts":"2026-06-05T09:41:05.267Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"7265fae5-1c6d-4609-bed7-2155c09c2aa3"}
{"level":"info","ts":"2026-06-05T09:41:05.267Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"7265fae5-1c6d-4609-bed7-2155c09c2aa3","1 head service found":"long-running-qkmps-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:05.267Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"7265fae5-1c6d-4609-bed7-2155c09c2aa3","error":"head certificate secret ray-head-secret-long-running-qkmps not found"}
{"level":"error","ts":"2026-06-05T09:41:05.267Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"7265fae5-1c6d-4609-bed7-2155c09c2aa3","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qkmps not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:05.267Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"7265fae5-1c6d-4609-bed7-2155c09c2aa3","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qkmps not found"}
{"level":"info","ts":"2026-06-05T09:41:05.267Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"7265fae5-1c6d-4609-bed7-2155c09c2aa3","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-qkmps","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:05.267Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"7265fae5-1c6d-4609-bed7-2155c09c2aa3","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-qkmps","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:05.267Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"7265fae5-1c6d-4609-bed7-2155c09c2aa3","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-qkmps","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:05.267Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"7265fae5-1c6d-4609-bed7-2155c09c2aa3"}
{"level":"error","ts":"2026-06-05T09:41:05.267Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"7265fae5-1c6d-4609-bed7-2155c09c2aa3","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qkmps not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:05.588Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"522f0641-9e8f-47cd-94b4-b0be53f6ee1f"}
{"level":"info","ts":"2026-06-05T09:41:05.588Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"522f0641-9e8f-47cd-94b4-b0be53f6ee1f"}
{"level":"info","ts":"2026-06-05T09:41:05.589Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"522f0641-9e8f-47cd-94b4-b0be53f6ee1f","1 head service found":"long-running-qkmps-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:05.589Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"522f0641-9e8f-47cd-94b4-b0be53f6ee1f","error":"head certificate secret ray-head-secret-long-running-qkmps not found"}
{"level":"error","ts":"2026-06-05T09:41:05.589Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"522f0641-9e8f-47cd-94b4-b0be53f6ee1f","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qkmps not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:05.589Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"522f0641-9e8f-47cd-94b4-b0be53f6ee1f","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qkmps not found"}
{"level":"info","ts":"2026-06-05T09:41:05.589Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"522f0641-9e8f-47cd-94b4-b0be53f6ee1f","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-qkmps","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:05.589Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"522f0641-9e8f-47cd-94b4-b0be53f6ee1f","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-qkmps","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:05.589Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"522f0641-9e8f-47cd-94b4-b0be53f6ee1f","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-qkmps","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:05.589Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"522f0641-9e8f-47cd-94b4-b0be53f6ee1f"}
{"level":"error","ts":"2026-06-05T09:41:05.589Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"522f0641-9e8f-47cd-94b4-b0be53f6ee1f","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qkmps not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:06.229Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"aae32839-848a-4556-b601-2613b39b1941"}
{"level":"info","ts":"2026-06-05T09:41:06.229Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"aae32839-848a-4556-b601-2613b39b1941"}
{"level":"info","ts":"2026-06-05T09:41:06.229Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"aae32839-848a-4556-b601-2613b39b1941","1 head service found":"long-running-qkmps-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:06.229Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"aae32839-848a-4556-b601-2613b39b1941","error":"head certificate secret ray-head-secret-long-running-qkmps not found"}
{"level":"error","ts":"2026-06-05T09:41:06.229Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"aae32839-848a-4556-b601-2613b39b1941","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qkmps not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:06.229Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"aae32839-848a-4556-b601-2613b39b1941","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qkmps not found"}
{"level":"info","ts":"2026-06-05T09:41:06.229Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"aae32839-848a-4556-b601-2613b39b1941","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-qkmps","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:06.229Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"aae32839-848a-4556-b601-2613b39b1941","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-qkmps","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:06.229Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"aae32839-848a-4556-b601-2613b39b1941","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-qkmps","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:06.229Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"aae32839-848a-4556-b601-2613b39b1941"}
{"level":"error","ts":"2026-06-05T09:41:06.229Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"aae32839-848a-4556-b601-2613b39b1941","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qkmps not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:07.021Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"stop","namespace":"test-ns-l4cdh"},"reconcileID":"bb7a3bd0-88c6-43d2-8c6f-2eed4169356a"}
{"level":"info","ts":"2026-06-05T09:41:07.511Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"5c5dd89c-9e5d-4a65-a32b-38269c243a9c"}
{"level":"info","ts":"2026-06-05T09:41:07.511Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"5c5dd89c-9e5d-4a65-a32b-38269c243a9c"}
{"level":"info","ts":"2026-06-05T09:41:07.511Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"5c5dd89c-9e5d-4a65-a32b-38269c243a9c","1 head service found":"long-running-qkmps-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:07.511Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"5c5dd89c-9e5d-4a65-a32b-38269c243a9c"}
{"level":"info","ts":"2026-06-05T09:41:07.511Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"5c5dd89c-9e5d-4a65-a32b-38269c243a9c","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:41:07.511Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"5c5dd89c-9e5d-4a65-a32b-38269c243a9c"}
{"level":"info","ts":"2026-06-05T09:41:07.511Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"5c5dd89c-9e5d-4a65-a32b-38269c243a9c","mode":"IntegratedOAuth","cluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:07.511Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"5c5dd89c-9e5d-4a65-a32b-38269c243a9c","cluster":"long-running-qkmps","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:41:07.511Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"5c5dd89c-9e5d-4a65-a32b-38269c243a9c","cluster":"long-running-qkmps","authType":"IntegratedOAuth","serviceAccount":"long-running-qkmps-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-05T09:41:07.511Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"5c5dd89c-9e5d-4a65-a32b-38269c243a9c","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"long-running-qkmps","ray.io/group":"headgroup","ray.io/identifier":"long-running-qkmps-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-05T09:41:07.511Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"5c5dd89c-9e5d-4a65-a32b-38269c243a9c","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"long-running-qkmps-head-svc.test-ns-t5s4r.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-05T09:41:07.511Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"5c5dd89c-9e5d-4a65-a32b-38269c243a9c","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-qkmps-head-svc.test-ns-t5s4r.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-05T09:41:07.511Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"5c5dd89c-9e5d-4a65-a32b-38269c243a9c","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-qkmps-head-svc.test-ns-t5s4r.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-05T09:41:07.511Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"5c5dd89c-9e5d-4a65-a32b-38269c243a9c","enabled":true}
{"level":"info","ts":"2026-06-05T09:41:07.531Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"5c5dd89c-9e5d-4a65-a32b-38269c243a9c","name":"long-running-qkmps-head-r2zkr"}
{"level":"info","ts":"2026-06-05T09:41:07.531Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"5c5dd89c-9e5d-4a65-a32b-38269c243a9c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:41:07.531Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"5c5dd89c-9e5d-4a65-a32b-38269c243a9c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:07.531Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"5c5dd89c-9e5d-4a65-a32b-38269c243a9c","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-05T09:41:07.531Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"5c5dd89c-9e5d-4a65-a32b-38269c243a9c","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:07.531Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"5c5dd89c-9e5d-4a65-a32b-38269c243a9c","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-05T09:41:07.531Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"5c5dd89c-9e5d-4a65-a32b-38269c243a9c"}
{"level":"info","ts":"2026-06-05T09:41:07.531Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"5c5dd89c-9e5d-4a65-a32b-38269c243a9c","nodeType":"worker","rayStartParams":{"address":"long-running-qkmps-head-svc.test-ns-t5s4r.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-05T09:41:07.531Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"5c5dd89c-9e5d-4a65-a32b-38269c243a9c","rayStartCmd":"ray start  --address=long-running-qkmps-head-svc.test-ns-t5s4r.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:41:07.531Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"5c5dd89c-9e5d-4a65-a32b-38269c243a9c","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-qkmps-head-svc.test-ns-t5s4r.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:41:07.531Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"5c5dd89c-9e5d-4a65-a32b-38269c243a9c","enabled":true}
{"level":"info","ts":"2026-06-05T09:41:07.546Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"5c5dd89c-9e5d-4a65-a32b-38269c243a9c","name":"long-running-qkmps-small-group-worker-pvt7t"}
{"level":"info","ts":"2026-06-05T09:41:07.546Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"5c5dd89c-9e5d-4a65-a32b-38269c243a9c","podName":"long-running-qkmps-head-r2zkr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:41:07.546Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"5c5dd89c-9e5d-4a65-a32b-38269c243a9c","podName":"long-running-qkmps-head-r2zkr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:41:07.546Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"5c5dd89c-9e5d-4a65-a32b-38269c243a9c","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"long-running-qkmps-head-svc"},"newHead":{"podName":"long-running-qkmps-head-r2zkr","serviceName":"long-running-qkmps-head-svc"}}
{"level":"info","ts":"2026-06-05T09:41:07.546Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"5c5dd89c-9e5d-4a65-a32b-38269c243a9c","name":"long-running-qkmps","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:41:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-qkmps-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:41:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:41:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-qkmps-head-r2zkr","serviceName":"long-running-qkmps-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:41:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:41:07.561Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"f49179bf-335e-45be-8aee-6b50e3d3de40"}
{"level":"info","ts":"2026-06-05T09:41:07.561Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"f49179bf-335e-45be-8aee-6b50e3d3de40"}
{"level":"info","ts":"2026-06-05T09:41:07.561Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"f49179bf-335e-45be-8aee-6b50e3d3de40","1 head service found":"long-running-qkmps-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:07.561Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"f49179bf-335e-45be-8aee-6b50e3d3de40","Found 1 head Pod":"long-running-qkmps-head-r2zkr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:41:07.561Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"f49179bf-335e-45be-8aee-6b50e3d3de40","head Pod":"long-running-qkmps-head-r2zkr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qkmps-head-r2zkr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:07.561Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"f49179bf-335e-45be-8aee-6b50e3d3de40","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:41:07.561Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"f49179bf-335e-45be-8aee-6b50e3d3de40","worker Pod":"long-running-qkmps-small-group-worker-pvt7t","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qkmps-small-group-worker-pvt7t. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:07.561Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"f49179bf-335e-45be-8aee-6b50e3d3de40","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:07.561Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"f49179bf-335e-45be-8aee-6b50e3d3de40","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:41:07.561Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"f49179bf-335e-45be-8aee-6b50e3d3de40","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:07.562Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"f49179bf-335e-45be-8aee-6b50e3d3de40","podName":"long-running-qkmps-head-r2zkr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:41:07.562Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"f49179bf-335e-45be-8aee-6b50e3d3de40","podName":"long-running-qkmps-head-r2zkr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:41:07.562Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"f49179bf-335e-45be-8aee-6b50e3d3de40","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"long-running-qkmps-head-svc"},"newHead":{"podName":"long-running-qkmps-head-r2zkr","serviceName":"long-running-qkmps-head-svc"}}
{"level":"info","ts":"2026-06-05T09:41:07.562Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"f49179bf-335e-45be-8aee-6b50e3d3de40","name":"long-running-qkmps","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:41:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-qkmps-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:41:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:41:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-qkmps-head-r2zkr","serviceName":"long-running-qkmps-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:41:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:41:07.563Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"e3ab1250-5328-436b-98ae-9bacff88f375","cluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:07.563Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"6dbb29f8-316e-4ad4-a495-8efc3abbdb46","rayCluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:07.563Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"6dbb29f8-316e-4ad4-a495-8efc3abbdb46","rayCluster":"long-running-qkmps","podIPs":[]}
{"level":"info","ts":"2026-06-05T09:41:07.563Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"5fc8dc48-7aee-4c14-b876-6df51b1a45d1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:07.563Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"5fc8dc48-7aee-4c14-b876-6df51b1a45d1","name":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:07.563Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"5fc8dc48-7aee-4c14-b876-6df51b1a45d1","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:07.563Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"5fc8dc48-7aee-4c14-b876-6df51b1a45d1"}
{"level":"info","ts":"2026-06-05T09:41:07.563Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"5fc8dc48-7aee-4c14-b876-6df51b1a45d1","RayCluster":"long-running-qkmps","State":""}
{"level":"info","ts":"2026-06-05T09:41:07.570Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"f49179bf-335e-45be-8aee-6b50e3d3de40","name":"long-running-qkmps","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-qkmps\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-t5s4r","name":"long-running-qkmps"}}
{"level":"info","ts":"2026-06-05T09:41:07.605Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"f49179bf-335e-45be-8aee-6b50e3d3de40"}
{"level":"error","ts":"2026-06-05T09:41:07.605Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"f49179bf-335e-45be-8aee-6b50e3d3de40","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-qkmps\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:07.606Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"460b8404-1bf4-4529-beed-8d22a37d5dc2"}
{"level":"info","ts":"2026-06-05T09:41:07.606Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"460b8404-1bf4-4529-beed-8d22a37d5dc2"}
{"level":"info","ts":"2026-06-05T09:41:07.606Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"460b8404-1bf4-4529-beed-8d22a37d5dc2","1 head service found":"long-running-qkmps-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:07.606Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"460b8404-1bf4-4529-beed-8d22a37d5dc2","Found 1 head Pod":"long-running-qkmps-head-r2zkr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:41:07.606Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"460b8404-1bf4-4529-beed-8d22a37d5dc2","head Pod":"long-running-qkmps-head-r2zkr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qkmps-head-r2zkr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:07.606Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"460b8404-1bf4-4529-beed-8d22a37d5dc2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:41:07.606Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"460b8404-1bf4-4529-beed-8d22a37d5dc2","worker Pod":"long-running-qkmps-small-group-worker-pvt7t","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qkmps-small-group-worker-pvt7t. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:07.606Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"460b8404-1bf4-4529-beed-8d22a37d5dc2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:07.606Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"460b8404-1bf4-4529-beed-8d22a37d5dc2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:41:07.606Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"460b8404-1bf4-4529-beed-8d22a37d5dc2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:07.606Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"460b8404-1bf4-4529-beed-8d22a37d5dc2","podName":"long-running-qkmps-head-r2zkr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:41:07.606Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"460b8404-1bf4-4529-beed-8d22a37d5dc2","podName":"long-running-qkmps-head-r2zkr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:41:07.606Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"460b8404-1bf4-4529-beed-8d22a37d5dc2","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:41:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:41:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-05T09:41:07.606Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"460b8404-1bf4-4529-beed-8d22a37d5dc2","name":"long-running-qkmps","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:41:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-qkmps-head-r2zkr","serviceName":"long-running-qkmps-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:41:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:41:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-qkmps-head-r2zkr","serviceName":"long-running-qkmps-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:41:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:41:07.706Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"e3ab1250-5328-436b-98ae-9bacff88f375","name":"long-running-qkmps-head"}
{"level":"info","ts":"2026-06-05T09:41:07.706Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"87436939-76a2-4985-9837-81cfdc35350d"}
{"level":"info","ts":"2026-06-05T09:41:07.706Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"87436939-76a2-4985-9837-81cfdc35350d"}
{"level":"info","ts":"2026-06-05T09:41:07.706Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"87436939-76a2-4985-9837-81cfdc35350d","1 head service found":"long-running-qkmps-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:07.706Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"9af4f795-5289-4b28-84b6-0b4c0c9e304c","rayCluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:07.706Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"87436939-76a2-4985-9837-81cfdc35350d","Found 1 head Pod":"long-running-qkmps-head-r2zkr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:41:07.706Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"87436939-76a2-4985-9837-81cfdc35350d","head Pod":"long-running-qkmps-head-r2zkr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qkmps-head-r2zkr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:07.706Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"87436939-76a2-4985-9837-81cfdc35350d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:41:07.706Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"87436939-76a2-4985-9837-81cfdc35350d","worker Pod":"long-running-qkmps-small-group-worker-pvt7t","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qkmps-small-group-worker-pvt7t. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:07.706Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"87436939-76a2-4985-9837-81cfdc35350d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:07.706Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"87436939-76a2-4985-9837-81cfdc35350d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:41:07.706Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"87436939-76a2-4985-9837-81cfdc35350d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:07.706Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"f87009fa-478d-431f-8096-7b7d1ff8042f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:07.706Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"f87009fa-478d-431f-8096-7b7d1ff8042f","name":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:07.706Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"f87009fa-478d-431f-8096-7b7d1ff8042f","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:07.706Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"9af4f795-5289-4b28-84b6-0b4c0c9e304c","rayCluster":"long-running-qkmps","podIPs":[]}
{"level":"info","ts":"2026-06-05T09:41:07.706Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"87436939-76a2-4985-9837-81cfdc35350d","podName":"long-running-qkmps-head-r2zkr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:41:07.707Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"87436939-76a2-4985-9837-81cfdc35350d","podName":"long-running-qkmps-head-r2zkr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:41:07.707Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"f87009fa-478d-431f-8096-7b7d1ff8042f"}
{"level":"info","ts":"2026-06-05T09:41:07.707Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"f87009fa-478d-431f-8096-7b7d1ff8042f","RayCluster":"long-running-qkmps","State":""}
{"level":"info","ts":"2026-06-05T09:41:07.707Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"87436939-76a2-4985-9837-81cfdc35350d","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:41:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:41:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-05T09:41:07.707Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"87436939-76a2-4985-9837-81cfdc35350d","name":"long-running-qkmps","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:41:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-qkmps-head-r2zkr","serviceName":"long-running-qkmps-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:41:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:41:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-qkmps-head-r2zkr","serviceName":"long-running-qkmps-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:41:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:41:07.713Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"87436939-76a2-4985-9837-81cfdc35350d","name":"long-running-qkmps","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-qkmps\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-t5s4r","name":"long-running-qkmps"}}
{"level":"info","ts":"2026-06-05T09:41:07.713Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"87436939-76a2-4985-9837-81cfdc35350d"}
{"level":"error","ts":"2026-06-05T09:41:07.713Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"87436939-76a2-4985-9837-81cfdc35350d","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-qkmps\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:07.716Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"e3ab1250-5328-436b-98ae-9bacff88f375","name":"long-running-qkmps-workers"}
{"level":"info","ts":"2026-06-05T09:41:07.716Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"e3ab1250-5328-436b-98ae-9bacff88f375","cluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:07.716Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"abb14bda-c261-4f5d-8140-0fcab2e2545d","cluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:07.719Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"6f8e06df-0864-4628-9dab-89b5cfd8e180"}
{"level":"info","ts":"2026-06-05T09:41:07.719Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"6f8e06df-0864-4628-9dab-89b5cfd8e180"}
{"level":"info","ts":"2026-06-05T09:41:07.719Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"6f8e06df-0864-4628-9dab-89b5cfd8e180","1 head service found":"long-running-qkmps-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:07.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"6f8e06df-0864-4628-9dab-89b5cfd8e180","Found 1 head Pod":"long-running-qkmps-head-r2zkr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:41:07.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"6f8e06df-0864-4628-9dab-89b5cfd8e180","head Pod":"long-running-qkmps-head-r2zkr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qkmps-head-r2zkr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:07.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"6f8e06df-0864-4628-9dab-89b5cfd8e180","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:41:07.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"6f8e06df-0864-4628-9dab-89b5cfd8e180","worker Pod":"long-running-qkmps-small-group-worker-pvt7t","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qkmps-small-group-worker-pvt7t. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:07.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"6f8e06df-0864-4628-9dab-89b5cfd8e180","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:07.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"6f8e06df-0864-4628-9dab-89b5cfd8e180","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:41:07.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"6f8e06df-0864-4628-9dab-89b5cfd8e180","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:07.719Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"6f8e06df-0864-4628-9dab-89b5cfd8e180","podName":"long-running-qkmps-head-r2zkr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:41:07.719Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"6f8e06df-0864-4628-9dab-89b5cfd8e180","podName":"long-running-qkmps-head-r2zkr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:41:07.719Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"6f8e06df-0864-4628-9dab-89b5cfd8e180","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:41:07.719Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"6f8e06df-0864-4628-9dab-89b5cfd8e180","seconds":300}
{"level":"info","ts":"2026-06-05T09:41:07.729Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"abb14bda-c261-4f5d-8140-0fcab2e2545d","name":"long-running-qkmps-head"}
{"level":"info","ts":"2026-06-05T09:41:07.734Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"5b197a15-3f9d-4f7b-b7dc-44f2d6150922","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:07.734Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"5b197a15-3f9d-4f7b-b7dc-44f2d6150922","name":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:07.734Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"5b197a15-3f9d-4f7b-b7dc-44f2d6150922","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:07.734Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"5b197a15-3f9d-4f7b-b7dc-44f2d6150922"}
{"level":"info","ts":"2026-06-05T09:41:07.734Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"5b197a15-3f9d-4f7b-b7dc-44f2d6150922","RayCluster":"long-running-qkmps","State":""}
{"level":"info","ts":"2026-06-05T09:41:07.740Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"abb14bda-c261-4f5d-8140-0fcab2e2545d","name":"long-running-qkmps-workers"}
{"level":"info","ts":"2026-06-05T09:41:07.740Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"abb14bda-c261-4f5d-8140-0fcab2e2545d","cluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:07.994Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"b2884df8-9f2b-4b28-bd8d-a02fb6501f5f"}
{"level":"info","ts":"2026-06-05T09:41:07.995Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"b2884df8-9f2b-4b28-bd8d-a02fb6501f5f"}
{"level":"info","ts":"2026-06-05T09:41:07.995Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"b2884df8-9f2b-4b28-bd8d-a02fb6501f5f","1 head service found":"long-running-qkmps-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:07.995Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"b2884df8-9f2b-4b28-bd8d-a02fb6501f5f","Found 1 head Pod":"long-running-qkmps-head-r2zkr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:41:07.995Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"b2884df8-9f2b-4b28-bd8d-a02fb6501f5f","head Pod":"long-running-qkmps-head-r2zkr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qkmps-head-r2zkr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:07.995Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"b2884df8-9f2b-4b28-bd8d-a02fb6501f5f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:41:07.995Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"b2884df8-9f2b-4b28-bd8d-a02fb6501f5f","worker Pod":"long-running-qkmps-small-group-worker-pvt7t","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qkmps-small-group-worker-pvt7t. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:07.995Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"b2884df8-9f2b-4b28-bd8d-a02fb6501f5f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:07.995Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"b2884df8-9f2b-4b28-bd8d-a02fb6501f5f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:41:07.995Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"b2884df8-9f2b-4b28-bd8d-a02fb6501f5f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:07.995Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"b2884df8-9f2b-4b28-bd8d-a02fb6501f5f","podName":"long-running-qkmps-head-r2zkr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:41:07.995Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"b2884df8-9f2b-4b28-bd8d-a02fb6501f5f","podName":"long-running-qkmps-head-r2zkr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:41:07.995Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"b2884df8-9f2b-4b28-bd8d-a02fb6501f5f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:41:07.995Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"b2884df8-9f2b-4b28-bd8d-a02fb6501f5f","seconds":300}
{"level":"info","ts":"2026-06-05T09:41:07.997Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"de954394-3603-4bb2-b6a2-0c691ea7497e"}
{"level":"info","ts":"2026-06-05T09:41:07.997Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"de954394-3603-4bb2-b6a2-0c691ea7497e"}
{"level":"info","ts":"2026-06-05T09:41:07.997Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"de954394-3603-4bb2-b6a2-0c691ea7497e","1 head service found":"long-running-qkmps-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:07.997Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"de954394-3603-4bb2-b6a2-0c691ea7497e","Found 1 head Pod":"long-running-qkmps-head-r2zkr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:41:07.997Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"de954394-3603-4bb2-b6a2-0c691ea7497e","head Pod":"long-running-qkmps-head-r2zkr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qkmps-head-r2zkr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:07.997Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"de954394-3603-4bb2-b6a2-0c691ea7497e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:41:07.997Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"de954394-3603-4bb2-b6a2-0c691ea7497e","worker Pod":"long-running-qkmps-small-group-worker-pvt7t","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qkmps-small-group-worker-pvt7t. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:07.997Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"de954394-3603-4bb2-b6a2-0c691ea7497e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:07.997Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"de954394-3603-4bb2-b6a2-0c691ea7497e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:41:07.997Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"de954394-3603-4bb2-b6a2-0c691ea7497e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:07.998Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"de954394-3603-4bb2-b6a2-0c691ea7497e","podName":"long-running-qkmps-head-r2zkr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:41:07.998Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"de954394-3603-4bb2-b6a2-0c691ea7497e","podName":"long-running-qkmps-head-r2zkr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:41:07.998Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"de954394-3603-4bb2-b6a2-0c691ea7497e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:41:07.998Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"de954394-3603-4bb2-b6a2-0c691ea7497e","seconds":300}
{"level":"info","ts":"2026-06-05T09:41:08.147Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"b1959579-e622-4f7c-8440-179a82693fd6"}
{"level":"info","ts":"2026-06-05T09:41:08.147Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"b1959579-e622-4f7c-8440-179a82693fd6"}
{"level":"info","ts":"2026-06-05T09:41:08.147Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"b1959579-e622-4f7c-8440-179a82693fd6","1 head service found":"long-running-qkmps-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:08.147Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"b1959579-e622-4f7c-8440-179a82693fd6","Found 1 head Pod":"long-running-qkmps-head-r2zkr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:41:08.147Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"b1959579-e622-4f7c-8440-179a82693fd6","head Pod":"long-running-qkmps-head-r2zkr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qkmps-head-r2zkr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:08.147Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"b1959579-e622-4f7c-8440-179a82693fd6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:41:08.147Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"b1959579-e622-4f7c-8440-179a82693fd6","worker Pod":"long-running-qkmps-small-group-worker-pvt7t","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qkmps-small-group-worker-pvt7t. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:08.147Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"b1959579-e622-4f7c-8440-179a82693fd6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:08.147Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"b1959579-e622-4f7c-8440-179a82693fd6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:41:08.147Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"b1959579-e622-4f7c-8440-179a82693fd6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:08.147Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"b1959579-e622-4f7c-8440-179a82693fd6","podName":"long-running-qkmps-head-r2zkr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:41:08.147Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"b1959579-e622-4f7c-8440-179a82693fd6","podName":"long-running-qkmps-head-r2zkr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:41:08.147Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"b1959579-e622-4f7c-8440-179a82693fd6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:41:08.147Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"b1959579-e622-4f7c-8440-179a82693fd6","seconds":300}
{"level":"info","ts":"2026-06-05T09:41:08.666Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"e3f3e5c9-2906-4ed1-a87a-422c062e0811"}
{"level":"info","ts":"2026-06-05T09:41:08.667Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"e3f3e5c9-2906-4ed1-a87a-422c062e0811"}
{"level":"info","ts":"2026-06-05T09:41:08.667Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"e3f3e5c9-2906-4ed1-a87a-422c062e0811","1 head service found":"long-running-qkmps-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:08.667Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"e3f3e5c9-2906-4ed1-a87a-422c062e0811","Found 1 head Pod":"long-running-qkmps-head-r2zkr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:41:08.667Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"e3f3e5c9-2906-4ed1-a87a-422c062e0811","head Pod":"long-running-qkmps-head-r2zkr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qkmps-head-r2zkr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:08.667Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"e3f3e5c9-2906-4ed1-a87a-422c062e0811","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:41:08.667Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"e3f3e5c9-2906-4ed1-a87a-422c062e0811","worker Pod":"long-running-qkmps-small-group-worker-pvt7t","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qkmps-small-group-worker-pvt7t. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:08.667Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"e3f3e5c9-2906-4ed1-a87a-422c062e0811","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:08.667Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"e3f3e5c9-2906-4ed1-a87a-422c062e0811","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:41:08.667Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"e3f3e5c9-2906-4ed1-a87a-422c062e0811","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:08.667Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"e3f3e5c9-2906-4ed1-a87a-422c062e0811","podName":"long-running-qkmps-head-r2zkr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:41:08.668Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"e3f3e5c9-2906-4ed1-a87a-422c062e0811","podName":"long-running-qkmps-head-r2zkr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:41:08.668Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"e3f3e5c9-2906-4ed1-a87a-422c062e0811","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"long-running-qkmps-head-r2zkr","serviceName":"long-running-qkmps-head-svc"},"newHead":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"long-running-qkmps-head-r2zkr","serviceName":"long-running-qkmps-head-svc"}}
{"level":"info","ts":"2026-06-05T09:41:08.668Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"e3f3e5c9-2906-4ed1-a87a-422c062e0811","name":"long-running-qkmps","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:41:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-qkmps-head-r2zkr","serviceName":"long-running-qkmps-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:41:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:41:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"long-running-qkmps-head-r2zkr","serviceName":"long-running-qkmps-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:41:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:41:08.706Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"e98b6994-1434-4858-b284-8b66cab35b5d","rayCluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:08.706Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"e98b6994-1434-4858-b284-8b66cab35b5d","pod":"long-running-qkmps-head-r2zkr","ip":"10.133.0.38"}
{"level":"info","ts":"2026-06-05T09:41:08.706Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"e98b6994-1434-4858-b284-8b66cab35b5d","pod":"long-running-qkmps-small-group-worker-pvt7t","ip":"10.132.0.18"}
{"level":"info","ts":"2026-06-05T09:41:08.706Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"98bd6525-2f56-4a35-a098-02e042900f0c","cluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:08.707Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"ed2ab7c3-b44a-4ab6-ab42-cbd03a052df3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:08.707Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"ed2ab7c3-b44a-4ab6-ab42-cbd03a052df3","name":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:08.707Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"ed2ab7c3-b44a-4ab6-ab42-cbd03a052df3","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:08.707Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"ed2ab7c3-b44a-4ab6-ab42-cbd03a052df3"}
{"level":"info","ts":"2026-06-05T09:41:08.707Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"ed2ab7c3-b44a-4ab6-ab42-cbd03a052df3","RayCluster":"long-running-qkmps","State":""}
{"level":"info","ts":"2026-06-05T09:41:08.717Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"e98b6994-1434-4858-b284-8b66cab35b5d","certificate":"ray-head-cert-long-running-qkmps","dnsNames":["long-running-qkmps-head-svc","localhost","long-running-qkmps-head-svc.test-ns-t5s4r.svc","long-running-qkmps-head-svc.test-ns-t5s4r.svc.cluster.local"],"ipAddresses":["10.133.0.38","10.132.0.18","127.0.0.1"]}
{"level":"info","ts":"2026-06-05T09:41:08.730Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"98bd6525-2f56-4a35-a098-02e042900f0c","name":"long-running-qkmps-head"}
{"level":"info","ts":"2026-06-05T09:41:08.810Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"98bd6525-2f56-4a35-a098-02e042900f0c","name":"long-running-qkmps-workers"}
{"level":"info","ts":"2026-06-05T09:41:08.810Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"98bd6525-2f56-4a35-a098-02e042900f0c","cluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:08.816Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"e98b6994-1434-4858-b284-8b66cab35b5d","certificate":"ray-worker-cert-long-running-qkmps","dnsNames":["long-running-qkmps-worker-svc","localhost","long-running-qkmps-small-group","long-running-qkmps-worker-svc.test-ns-t5s4r.svc","long-running-qkmps-worker-svc.test-ns-t5s4r.svc.cluster.local","*.long-running-qkmps-worker-svc.test-ns-t5s4r.svc","*.long-running-qkmps-worker-svc.test-ns-t5s4r.svc.cluster.local","*-worker-*.test-ns-t5s4r.svc","*-worker-*.test-ns-t5s4r.svc.cluster.local"],"ipAddresses":["10.133.0.38","10.132.0.18","127.0.0.1"]}
{"level":"info","ts":"2026-06-05T09:41:08.816Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"e98b6994-1434-4858-b284-8b66cab35b5d"}
{"level":"info","ts":"2026-06-05T09:41:09.632Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"72c163bb-cbe5-46d5-bef8-74d6f81c17fd"}
{"level":"info","ts":"2026-06-05T09:41:09.632Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"72c163bb-cbe5-46d5-bef8-74d6f81c17fd"}
{"level":"info","ts":"2026-06-05T09:41:09.632Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"72c163bb-cbe5-46d5-bef8-74d6f81c17fd","1 head service found":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:09.632Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"72c163bb-cbe5-46d5-bef8-74d6f81c17fd","Found 1 head Pod":"fail-d7vcr-head-8br8s","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:41:09.632Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"72c163bb-cbe5-46d5-bef8-74d6f81c17fd","head Pod":"fail-d7vcr-head-8br8s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-d7vcr-head-8br8s. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:09.632Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"72c163bb-cbe5-46d5-bef8-74d6f81c17fd","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:41:09.632Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"72c163bb-cbe5-46d5-bef8-74d6f81c17fd","worker Pod":"fail-d7vcr-small-group-worker-crtnw","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-d7vcr-small-group-worker-crtnw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:09.632Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"72c163bb-cbe5-46d5-bef8-74d6f81c17fd","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:09.632Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"72c163bb-cbe5-46d5-bef8-74d6f81c17fd","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:41:09.632Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"72c163bb-cbe5-46d5-bef8-74d6f81c17fd","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:09.633Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"72c163bb-cbe5-46d5-bef8-74d6f81c17fd","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:41:09.633Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"72c163bb-cbe5-46d5-bef8-74d6f81c17fd","seconds":300}
{"level":"info","ts":"2026-06-05T09:41:09.637Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"b8d6eeab-ec0a-4cf8-839b-f3e84410f751"}
{"level":"info","ts":"2026-06-05T09:41:09.637Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"b8d6eeab-ec0a-4cf8-839b-f3e84410f751"}
{"level":"info","ts":"2026-06-05T09:41:09.637Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"b8d6eeab-ec0a-4cf8-839b-f3e84410f751","1 head service found":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:09.637Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"b8d6eeab-ec0a-4cf8-839b-f3e84410f751","Found 1 head Pod":"fail-d7vcr-head-8br8s","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:41:09.637Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"b8d6eeab-ec0a-4cf8-839b-f3e84410f751","head Pod":"fail-d7vcr-head-8br8s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-d7vcr-head-8br8s. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:09.637Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"b8d6eeab-ec0a-4cf8-839b-f3e84410f751","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:41:09.637Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"b8d6eeab-ec0a-4cf8-839b-f3e84410f751","worker Pod":"fail-d7vcr-small-group-worker-crtnw","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-d7vcr-small-group-worker-crtnw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:09.637Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"b8d6eeab-ec0a-4cf8-839b-f3e84410f751","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:09.637Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"b8d6eeab-ec0a-4cf8-839b-f3e84410f751","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:41:09.637Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"b8d6eeab-ec0a-4cf8-839b-f3e84410f751","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:09.637Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"b8d6eeab-ec0a-4cf8-839b-f3e84410f751","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:41:09.637Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"b8d6eeab-ec0a-4cf8-839b-f3e84410f751","seconds":300}
{"level":"info","ts":"2026-06-05T09:41:09.659Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f614a361-bd6b-497f-9f39-dfb2fa475479"}
{"level":"info","ts":"2026-06-05T09:41:09.659Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f614a361-bd6b-497f-9f39-dfb2fa475479"}
{"level":"info","ts":"2026-06-05T09:41:09.659Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f614a361-bd6b-497f-9f39-dfb2fa475479","1 head service found":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:09.659Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f614a361-bd6b-497f-9f39-dfb2fa475479","Found 1 head Pod":"fail-d7vcr-head-8br8s","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:41:09.659Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f614a361-bd6b-497f-9f39-dfb2fa475479","head Pod":"fail-d7vcr-head-8br8s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-d7vcr-head-8br8s. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:09.659Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f614a361-bd6b-497f-9f39-dfb2fa475479","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:41:09.659Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f614a361-bd6b-497f-9f39-dfb2fa475479","worker Pod":"fail-d7vcr-small-group-worker-crtnw","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-d7vcr-small-group-worker-crtnw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:09.659Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f614a361-bd6b-497f-9f39-dfb2fa475479","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:09.659Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f614a361-bd6b-497f-9f39-dfb2fa475479","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:41:09.659Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f614a361-bd6b-497f-9f39-dfb2fa475479","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:09.659Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f614a361-bd6b-497f-9f39-dfb2fa475479","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:41:09.659Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f614a361-bd6b-497f-9f39-dfb2fa475479","seconds":300}
{"level":"info","ts":"2026-06-05T09:41:09.660Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"7e603658-f160-4360-9149-940ca54887ff"}
{"level":"info","ts":"2026-06-05T09:41:09.660Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"7e603658-f160-4360-9149-940ca54887ff"}
{"level":"info","ts":"2026-06-05T09:41:09.660Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"7e603658-f160-4360-9149-940ca54887ff","1 head service found":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:09.660Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"7e603658-f160-4360-9149-940ca54887ff","Found 1 head Pod":"fail-d7vcr-head-8br8s","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:41:09.660Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"7e603658-f160-4360-9149-940ca54887ff","head Pod":"fail-d7vcr-head-8br8s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-d7vcr-head-8br8s. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:09.660Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"7e603658-f160-4360-9149-940ca54887ff","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:41:09.660Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"7e603658-f160-4360-9149-940ca54887ff","worker Pod":"fail-d7vcr-small-group-worker-crtnw","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-d7vcr-small-group-worker-crtnw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:09.660Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"7e603658-f160-4360-9149-940ca54887ff","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:09.660Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"7e603658-f160-4360-9149-940ca54887ff","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:41:09.660Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"7e603658-f160-4360-9149-940ca54887ff","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:09.660Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"7e603658-f160-4360-9149-940ca54887ff","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:41:09.660Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"7e603658-f160-4360-9149-940ca54887ff","seconds":300}
{"level":"info","ts":"2026-06-05T09:41:10.321Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c5c52393-c18a-4696-b567-f504461f0e97"}
{"level":"info","ts":"2026-06-05T09:41:10.321Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c5c52393-c18a-4696-b567-f504461f0e97"}
{"level":"info","ts":"2026-06-05T09:41:10.321Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c5c52393-c18a-4696-b567-f504461f0e97","1 head service found":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:10.321Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c5c52393-c18a-4696-b567-f504461f0e97","Found 1 head Pod":"fail-d7vcr-head-8br8s","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:41:10.321Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c5c52393-c18a-4696-b567-f504461f0e97","head Pod":"fail-d7vcr-head-8br8s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-d7vcr-head-8br8s. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:10.321Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c5c52393-c18a-4696-b567-f504461f0e97","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:41:10.321Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c5c52393-c18a-4696-b567-f504461f0e97","worker Pod":"fail-d7vcr-small-group-worker-crtnw","shouldDelete":true,"reason":"The worker Pod fail-d7vcr-small-group-worker-crtnw status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"error","ts":"2026-06-05T09:41:10.326Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c5c52393-c18a-4696-b567-f504461f0e97","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"delete 1 unhealthy worker Pods","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:10.326Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c5c52393-c18a-4696-b567-f504461f0e97","error":"delete 1 unhealthy worker Pods"}
{"level":"info","ts":"2026-06-05T09:41:10.326Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c5c52393-c18a-4696-b567-f504461f0e97","oldReadyWorkerReplicas":1,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":1,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-05T09:41:10.326Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c5c52393-c18a-4696-b567-f504461f0e97","name":"fail-d7vcr","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:55Z","stateTransitionTimes":{"ready":"2026-06-05T09:39:55Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.16","serviceIP":"10.132.0.16","podName":"fail-d7vcr-head-8br8s","serviceName":"fail-d7vcr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:39:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:39:55Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:41:10Z","stateTransitionTimes":{"ready":"2026-06-05T09:39:55Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.16","serviceIP":"10.132.0.16","podName":"fail-d7vcr-head-8br8s","serviceName":"fail-d7vcr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:39:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:39:55Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:41:10.338Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c5c52393-c18a-4696-b567-f504461f0e97"}
{"level":"error","ts":"2026-06-05T09:41:10.338Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c5c52393-c18a-4696-b567-f504461f0e97","error":"delete 1 unhealthy worker Pods","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:10.339Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"283a46a8-3aa2-4aa8-800d-7f14700403f9","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:41:10.339Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"13df760a-e31f-46b8-94a9-bd7ff109f5ef","rayCluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:41:10.339Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"0589b454-fbea-4bae-85d0-7433dd0790e0","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:41:10.339Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"0589b454-fbea-4bae-85d0-7433dd0790e0","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-05 09:40:00 +0000 UTC","Now":"2026-06-05T09:41:10.339Z","ShutdownTime":"2026-06-05T09:40:00.000Z"}
{"level":"info","ts":"2026-06-05T09:41:10.339Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"13df760a-e31f-46b8-94a9-bd7ff109f5ef","pod":"fail-d7vcr-head-8br8s","ip":"10.132.0.16"}
{"level":"info","ts":"2026-06-05T09:41:10.339Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"13df760a-e31f-46b8-94a9-bd7ff109f5ef","pod":"fail-d7vcr-small-group-worker-crtnw","ip":"10.133.0.36"}
{"level":"info","ts":"2026-06-05T09:41:10.339Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"13df760a-e31f-46b8-94a9-bd7ff109f5ef","rayCluster":"fail-d7vcr","podIPs":["10.132.0.16","10.133.0.36"]}
{"level":"info","ts":"2026-06-05T09:41:10.344Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"b7c704da-c14e-4321-89d3-bb32773843e3"}
{"level":"info","ts":"2026-06-05T09:41:10.344Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"b7c704da-c14e-4321-89d3-bb32773843e3"}
{"level":"info","ts":"2026-06-05T09:41:10.344Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"b7c704da-c14e-4321-89d3-bb32773843e3","1 head service found":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:10.344Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"b7c704da-c14e-4321-89d3-bb32773843e3","Found 1 head Pod":"fail-d7vcr-head-8br8s","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:41:10.344Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"b7c704da-c14e-4321-89d3-bb32773843e3","head Pod":"fail-d7vcr-head-8br8s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-d7vcr-head-8br8s. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:10.344Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"b7c704da-c14e-4321-89d3-bb32773843e3","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-06-05T09:41:10.344Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"b7c704da-c14e-4321-89d3-bb32773843e3","podName":"fail-d7vcr-small-group-worker-crtnw","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-06-05T09:41:10.345Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"b7c704da-c14e-4321-89d3-bb32773843e3","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:41:10.345Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"b7c704da-c14e-4321-89d3-bb32773843e3","seconds":300}
{"level":"info","ts":"2026-06-05T09:41:10.345Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"283a46a8-3aa2-4aa8-800d-7f14700403f9","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:41:10.497Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"5bcd905e-c82c-4b7b-b7a2-7c08219f9aa9"}
{"level":"info","ts":"2026-06-05T09:41:10.497Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"5bcd905e-c82c-4b7b-b7a2-7c08219f9aa9"}
{"level":"info","ts":"2026-06-05T09:41:10.497Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"5bcd905e-c82c-4b7b-b7a2-7c08219f9aa9","1 head service found":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:10.497Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"5bcd905e-c82c-4b7b-b7a2-7c08219f9aa9","Found 1 head Pod":"fail-d7vcr-head-8br8s","Pod status":"Failed","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"&ContainerStateTerminated{ExitCode:1,Signal:0,Reason:Error,Message:,StartedAt:2026-06-05 09:39:38 +0000 UTC,FinishedAt:2026-06-05 09:41:10 +0000 UTC,ContainerID:cri-o://af2aaa6adb106c04122bb7a44e06c8d1294f9f7960c7b9cf4a332d95efdc0187,}"}
{"level":"info","ts":"2026-06-05T09:41:10.497Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"5bcd905e-c82c-4b7b-b7a2-7c08219f9aa9","head Pod":"fail-d7vcr-head-8br8s","shouldDelete":true,"reason":"The head Pod fail-d7vcr-head-8br8s status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"error","ts":"2026-06-05T09:41:10.501Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"5bcd905e-c82c-4b7b-b7a2-7c08219f9aa9","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"The head Pod fail-d7vcr-head-8br8s status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:10.501Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"5bcd905e-c82c-4b7b-b7a2-7c08219f9aa9","error":"The head Pod fail-d7vcr-head-8br8s status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"info","ts":"2026-06-05T09:41:10.501Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"5bcd905e-c82c-4b7b-b7a2-7c08219f9aa9","old conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:39:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:39:55Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:10Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:39:55Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-05T09:41:10.501Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"5bcd905e-c82c-4b7b-b7a2-7c08219f9aa9","name":"fail-d7vcr","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:41:10Z","stateTransitionTimes":{"ready":"2026-06-05T09:39:55Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.16","serviceIP":"10.132.0.16","podName":"fail-d7vcr-head-8br8s","serviceName":"fail-d7vcr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:39:50Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:39:55Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:41:10Z","stateTransitionTimes":{"ready":"2026-06-05T09:39:55Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.16","serviceIP":"10.132.0.16","podName":"fail-d7vcr-head-8br8s","serviceName":"fail-d7vcr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:10Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:39:55Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:41:10.514Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"5bcd905e-c82c-4b7b-b7a2-7c08219f9aa9"}
{"level":"error","ts":"2026-06-05T09:41:10.514Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"5bcd905e-c82c-4b7b-b7a2-7c08219f9aa9","error":"The head Pod fail-d7vcr-head-8br8s status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:10.514Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"c983b616-6dd0-4d84-bd7b-48c661f8671e","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:41:10.514Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"4d52fb20-2b8c-4f3a-9d57-d7cf144b946b","rayCluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:41:10.514Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"f48ae4d0-be14-420c-8928-1fa0b1b1aced","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:41:10.514Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"f48ae4d0-be14-420c-8928-1fa0b1b1aced","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-05 09:40:00 +0000 UTC","Now":"2026-06-05T09:41:10.514Z","ShutdownTime":"2026-06-05T09:40:00.000Z"}
{"level":"info","ts":"2026-06-05T09:41:10.514Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"4d52fb20-2b8c-4f3a-9d57-d7cf144b946b","pod":"fail-d7vcr-head-8br8s","ip":"10.132.0.16"}
{"level":"info","ts":"2026-06-05T09:41:10.514Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"4d52fb20-2b8c-4f3a-9d57-d7cf144b946b","pod":"fail-d7vcr-small-group-worker-crtnw","ip":"10.133.0.36"}
{"level":"info","ts":"2026-06-05T09:41:10.514Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"4d52fb20-2b8c-4f3a-9d57-d7cf144b946b","rayCluster":"fail-d7vcr","podIPs":["10.132.0.16","10.133.0.36"]}
{"level":"info","ts":"2026-06-05T09:41:10.519Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"e46fa0ee-641e-4567-a579-757fbd163eac"}
{"level":"info","ts":"2026-06-05T09:41:10.519Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"e46fa0ee-641e-4567-a579-757fbd163eac"}
{"level":"info","ts":"2026-06-05T09:41:10.519Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"e46fa0ee-641e-4567-a579-757fbd163eac","1 head service found":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:10.519Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"c983b616-6dd0-4d84-bd7b-48c661f8671e","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:41:10.519Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"e46fa0ee-641e-4567-a579-757fbd163eac","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-06-05T09:41:10.519Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"e46fa0ee-641e-4567-a579-757fbd163eac","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-06-05T09:41:10.519Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"e46fa0ee-641e-4567-a579-757fbd163eac","podName":"fail-d7vcr-head-8br8s","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-06-05T09:41:10.519Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"e46fa0ee-641e-4567-a579-757fbd163eac","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:41:10.519Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"e46fa0ee-641e-4567-a579-757fbd163eac","seconds":300}
{"level":"info","ts":"2026-06-05T09:41:10.676Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"444ff9c8-e847-4534-9a8a-26cd0055c92b"}
{"level":"info","ts":"2026-06-05T09:41:10.676Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"444ff9c8-e847-4534-9a8a-26cd0055c92b"}
{"level":"info","ts":"2026-06-05T09:41:10.676Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"444ff9c8-e847-4534-9a8a-26cd0055c92b","1 head service found":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:10.676Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"444ff9c8-e847-4534-9a8a-26cd0055c92b","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-06-05T09:41:10.676Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"444ff9c8-e847-4534-9a8a-26cd0055c92b","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-06-05T09:41:10.676Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"444ff9c8-e847-4534-9a8a-26cd0055c92b","podName":"fail-d7vcr-head-8br8s","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-06-05T09:41:10.677Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"444ff9c8-e847-4534-9a8a-26cd0055c92b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:41:10.677Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"444ff9c8-e847-4534-9a8a-26cd0055c92b","seconds":300}
{"level":"info","ts":"2026-06-05T09:41:10.679Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"9ac1be26-c877-48e5-82a1-b331207ddf1e"}
{"level":"info","ts":"2026-06-05T09:41:10.679Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"9ac1be26-c877-48e5-82a1-b331207ddf1e"}
{"level":"info","ts":"2026-06-05T09:41:10.679Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"9ac1be26-c877-48e5-82a1-b331207ddf1e","1 head service found":"long-running-qkmps-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:10.679Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"9ac1be26-c877-48e5-82a1-b331207ddf1e","Found 1 head Pod":"long-running-qkmps-head-r2zkr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:41:10.679Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"9ac1be26-c877-48e5-82a1-b331207ddf1e","head Pod":"long-running-qkmps-head-r2zkr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qkmps-head-r2zkr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:10.679Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"9ac1be26-c877-48e5-82a1-b331207ddf1e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:41:10.679Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"9ac1be26-c877-48e5-82a1-b331207ddf1e","worker Pod":"long-running-qkmps-small-group-worker-pvt7t","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qkmps-small-group-worker-pvt7t. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:10.679Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"9ac1be26-c877-48e5-82a1-b331207ddf1e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:10.679Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"9ac1be26-c877-48e5-82a1-b331207ddf1e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:41:10.679Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"9ac1be26-c877-48e5-82a1-b331207ddf1e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:10.679Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"9ac1be26-c877-48e5-82a1-b331207ddf1e","podName":"long-running-qkmps-head-r2zkr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:41:10.679Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"9ac1be26-c877-48e5-82a1-b331207ddf1e","podName":"long-running-qkmps-head-r2zkr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:41:10.679Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"9ac1be26-c877-48e5-82a1-b331207ddf1e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:41:10.679Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"9ac1be26-c877-48e5-82a1-b331207ddf1e","seconds":300}
{"level":"info","ts":"2026-06-05T09:41:10.682Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"e5dc295d-58bb-4ecd-be76-11cd34cc285a"}
{"level":"info","ts":"2026-06-05T09:41:10.682Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"e5dc295d-58bb-4ecd-be76-11cd34cc285a"}
{"level":"info","ts":"2026-06-05T09:41:10.682Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"e5dc295d-58bb-4ecd-be76-11cd34cc285a","1 head service found":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:10.682Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"e5dc295d-58bb-4ecd-be76-11cd34cc285a","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-06-05T09:41:10.682Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"e5dc295d-58bb-4ecd-be76-11cd34cc285a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:41:10.682Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"e5dc295d-58bb-4ecd-be76-11cd34cc285a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:10.682Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"e5dc295d-58bb-4ecd-be76-11cd34cc285a","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-05T09:41:10.682Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"e5dc295d-58bb-4ecd-be76-11cd34cc285a","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:10.682Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"e5dc295d-58bb-4ecd-be76-11cd34cc285a","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-05T09:41:10.682Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"e5dc295d-58bb-4ecd-be76-11cd34cc285a"}
{"level":"info","ts":"2026-06-05T09:41:10.682Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"e5dc295d-58bb-4ecd-be76-11cd34cc285a","nodeType":"worker","rayStartParams":{"address":"fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-05T09:41:10.682Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"e5dc295d-58bb-4ecd-be76-11cd34cc285a","rayStartCmd":"ray start  --address=fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:41:10.682Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"e5dc295d-58bb-4ecd-be76-11cd34cc285a","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:41:10.682Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"e5dc295d-58bb-4ecd-be76-11cd34cc285a","enabled":true}
{"level":"error","ts":"2026-06-05T09:41:10.686Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"e5dc295d-58bb-4ecd-be76-11cd34cc285a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"fail-d7vcr-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:10.686Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"e5dc295d-58bb-4ecd-be76-11cd34cc285a","error":"FailedCreateWorkerPod\npods \"fail-d7vcr-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated"}
{"level":"info","ts":"2026-06-05T09:41:10.686Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"e5dc295d-58bb-4ecd-be76-11cd34cc285a","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:10Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:39:55Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:10Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:39:55Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-05T09:41:10Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"fail-d7vcr-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated"}]}
{"level":"info","ts":"2026-06-05T09:41:10.686Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"e5dc295d-58bb-4ecd-be76-11cd34cc285a","name":"fail-d7vcr","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:41:10Z","stateTransitionTimes":{"ready":"2026-06-05T09:39:55Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.16","serviceIP":"10.132.0.16","podName":"fail-d7vcr-head-8br8s","serviceName":"fail-d7vcr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:10Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:39:55Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:41:10Z","stateTransitionTimes":{"ready":"2026-06-05T09:39:55Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.16","serviceIP":"10.132.0.16","podName":"fail-d7vcr-head-8br8s","serviceName":"fail-d7vcr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:10Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:39:55Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-05T09:41:10Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"fail-d7vcr-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:41:10.697Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"e5dc295d-58bb-4ecd-be76-11cd34cc285a"}
{"level":"error","ts":"2026-06-05T09:41:10.697Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"e5dc295d-58bb-4ecd-be76-11cd34cc285a","error":"FailedCreateWorkerPod\npods \"fail-d7vcr-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:10.698Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"c86c4bfd-1848-4c7d-8ea0-cc3ca8706799","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:41:10.698Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"a6348840-12bc-4c7c-b14f-9d12c07c4016","rayCluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:41:10.698Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"a6348840-12bc-4c7c-b14f-9d12c07c4016","pod":"fail-d7vcr-head-8br8s","ip":"10.132.0.16"}
{"level":"info","ts":"2026-06-05T09:41:10.698Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"92f681ea-cb1c-478a-82cd-41a4276391ea","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:41:10.698Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"92f681ea-cb1c-478a-82cd-41a4276391ea","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-05 09:40:00 +0000 UTC","Now":"2026-06-05T09:41:10.698Z","ShutdownTime":"2026-06-05T09:40:00.000Z"}
{"level":"info","ts":"2026-06-05T09:41:10.702Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"c86c4bfd-1848-4c7d-8ea0-cc3ca8706799","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:41:10.702Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f1956f76-4562-4ff5-aa8f-ea7029c76498"}
{"level":"info","ts":"2026-06-05T09:41:10.703Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f1956f76-4562-4ff5-aa8f-ea7029c76498"}
{"level":"info","ts":"2026-06-05T09:41:10.703Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f1956f76-4562-4ff5-aa8f-ea7029c76498","1 head service found":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:10.703Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f1956f76-4562-4ff5-aa8f-ea7029c76498","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-06-05T09:41:10.703Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f1956f76-4562-4ff5-aa8f-ea7029c76498","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:41:10.703Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f1956f76-4562-4ff5-aa8f-ea7029c76498","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:10.703Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f1956f76-4562-4ff5-aa8f-ea7029c76498","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-05T09:41:10.703Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f1956f76-4562-4ff5-aa8f-ea7029c76498","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:10.703Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f1956f76-4562-4ff5-aa8f-ea7029c76498","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-05T09:41:10.703Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f1956f76-4562-4ff5-aa8f-ea7029c76498"}
{"level":"info","ts":"2026-06-05T09:41:10.703Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f1956f76-4562-4ff5-aa8f-ea7029c76498","nodeType":"worker","rayStartParams":{"address":"fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-05T09:41:10.703Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f1956f76-4562-4ff5-aa8f-ea7029c76498","rayStartCmd":"ray start  --address=fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:41:10.703Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f1956f76-4562-4ff5-aa8f-ea7029c76498","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:41:10.703Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f1956f76-4562-4ff5-aa8f-ea7029c76498","enabled":true}
{"level":"error","ts":"2026-06-05T09:41:10.706Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f1956f76-4562-4ff5-aa8f-ea7029c76498","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"fail-d7vcr-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:10.706Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f1956f76-4562-4ff5-aa8f-ea7029c76498","error":"FailedCreateWorkerPod\npods \"fail-d7vcr-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated"}
{"level":"info","ts":"2026-06-05T09:41:10.706Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f1956f76-4562-4ff5-aa8f-ea7029c76498"}
{"level":"error","ts":"2026-06-05T09:41:10.706Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"f1956f76-4562-4ff5-aa8f-ea7029c76498","error":"FailedCreateWorkerPod\npods \"fail-d7vcr-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:10.708Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"a6348840-12bc-4c7c-b14f-9d12c07c4016","certificate":"ray-head-cert-fail-d7vcr","dnsNames":["fail-d7vcr-head-svc","localhost","fail-d7vcr-head-svc.test-ns-l4cdh.svc","fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local"],"ipAddresses":["10.132.0.16","127.0.0.1"]}
{"level":"info","ts":"2026-06-05T09:41:10.716Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"907bedf5-5fdb-4eda-98ec-eaa9c5a7d7fa"}
{"level":"info","ts":"2026-06-05T09:41:10.716Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"907bedf5-5fdb-4eda-98ec-eaa9c5a7d7fa"}
{"level":"info","ts":"2026-06-05T09:41:10.717Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"907bedf5-5fdb-4eda-98ec-eaa9c5a7d7fa","1 head service found":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:10.717Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"907bedf5-5fdb-4eda-98ec-eaa9c5a7d7fa","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-06-05T09:41:10.717Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"907bedf5-5fdb-4eda-98ec-eaa9c5a7d7fa","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:41:10.717Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"907bedf5-5fdb-4eda-98ec-eaa9c5a7d7fa","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:10.717Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"907bedf5-5fdb-4eda-98ec-eaa9c5a7d7fa","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-05T09:41:10.717Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"907bedf5-5fdb-4eda-98ec-eaa9c5a7d7fa","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:10.717Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"907bedf5-5fdb-4eda-98ec-eaa9c5a7d7fa","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-05T09:41:10.717Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"907bedf5-5fdb-4eda-98ec-eaa9c5a7d7fa"}
{"level":"info","ts":"2026-06-05T09:41:10.717Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"907bedf5-5fdb-4eda-98ec-eaa9c5a7d7fa","nodeType":"worker","rayStartParams":{"address":"fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-05T09:41:10.717Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"907bedf5-5fdb-4eda-98ec-eaa9c5a7d7fa","rayStartCmd":"ray start  --address=fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:41:10.717Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"907bedf5-5fdb-4eda-98ec-eaa9c5a7d7fa","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:41:10.717Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"907bedf5-5fdb-4eda-98ec-eaa9c5a7d7fa","enabled":true}
{"level":"error","ts":"2026-06-05T09:41:10.720Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"907bedf5-5fdb-4eda-98ec-eaa9c5a7d7fa","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"fail-d7vcr-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:10.720Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"907bedf5-5fdb-4eda-98ec-eaa9c5a7d7fa","error":"FailedCreateWorkerPod\npods \"fail-d7vcr-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated"}
{"level":"info","ts":"2026-06-05T09:41:10.721Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"907bedf5-5fdb-4eda-98ec-eaa9c5a7d7fa"}
{"level":"error","ts":"2026-06-05T09:41:10.721Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"907bedf5-5fdb-4eda-98ec-eaa9c5a7d7fa","error":"FailedCreateWorkerPod\npods \"fail-d7vcr-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:10.721Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"a6348840-12bc-4c7c-b14f-9d12c07c4016","certificate":"ray-worker-cert-fail-d7vcr","dnsNames":["fail-d7vcr-worker-svc","localhost","fail-d7vcr-small-group","fail-d7vcr-worker-svc.test-ns-l4cdh.svc","fail-d7vcr-worker-svc.test-ns-l4cdh.svc.cluster.local","*.fail-d7vcr-worker-svc.test-ns-l4cdh.svc","*.fail-d7vcr-worker-svc.test-ns-l4cdh.svc.cluster.local","*-worker-*.test-ns-l4cdh.svc","*-worker-*.test-ns-l4cdh.svc.cluster.local"],"ipAddresses":["10.132.0.16","127.0.0.1"]}
{"level":"info","ts":"2026-06-05T09:41:10.721Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"a6348840-12bc-4c7c-b14f-9d12c07c4016","rayCluster":"fail-d7vcr","podIPs":["10.132.0.16"]}
{"level":"info","ts":"2026-06-05T09:41:10.735Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"3783de0f-60d7-45ad-92fd-4e61ff496f1b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:10.735Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"3783de0f-60d7-45ad-92fd-4e61ff496f1b","name":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:10.735Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"3783de0f-60d7-45ad-92fd-4e61ff496f1b","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:10.735Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"3783de0f-60d7-45ad-92fd-4e61ff496f1b"}
{"level":"info","ts":"2026-06-05T09:41:10.735Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"3783de0f-60d7-45ad-92fd-4e61ff496f1b","RayCluster":"long-running-qkmps","State":""}
{"level":"info","ts":"2026-06-05T09:41:10.742Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"ca71fe43-cbcb-4c94-9e9a-d7603c36759c"}
{"level":"info","ts":"2026-06-05T09:41:10.742Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"ca71fe43-cbcb-4c94-9e9a-d7603c36759c"}
{"level":"info","ts":"2026-06-05T09:41:10.742Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"ca71fe43-cbcb-4c94-9e9a-d7603c36759c","1 head service found":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:10.742Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"ca71fe43-cbcb-4c94-9e9a-d7603c36759c","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-06-05T09:41:10.742Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"ca71fe43-cbcb-4c94-9e9a-d7603c36759c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:41:10.742Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"ca71fe43-cbcb-4c94-9e9a-d7603c36759c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:10.742Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"ca71fe43-cbcb-4c94-9e9a-d7603c36759c","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-05T09:41:10.742Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"ca71fe43-cbcb-4c94-9e9a-d7603c36759c","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:10.742Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"ca71fe43-cbcb-4c94-9e9a-d7603c36759c","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-05T09:41:10.742Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"ca71fe43-cbcb-4c94-9e9a-d7603c36759c"}
{"level":"info","ts":"2026-06-05T09:41:10.742Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"ca71fe43-cbcb-4c94-9e9a-d7603c36759c","nodeType":"worker","rayStartParams":{"address":"fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-05T09:41:10.742Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"ca71fe43-cbcb-4c94-9e9a-d7603c36759c","rayStartCmd":"ray start  --address=fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:41:10.742Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"ca71fe43-cbcb-4c94-9e9a-d7603c36759c","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:41:10.742Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"ca71fe43-cbcb-4c94-9e9a-d7603c36759c","enabled":true}
{"level":"error","ts":"2026-06-05T09:41:10.745Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"ca71fe43-cbcb-4c94-9e9a-d7603c36759c","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"fail-d7vcr-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:10.745Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"ca71fe43-cbcb-4c94-9e9a-d7603c36759c","error":"FailedCreateWorkerPod\npods \"fail-d7vcr-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated"}
{"level":"info","ts":"2026-06-05T09:41:10.745Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"ca71fe43-cbcb-4c94-9e9a-d7603c36759c"}
{"level":"error","ts":"2026-06-05T09:41:10.745Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"ca71fe43-cbcb-4c94-9e9a-d7603c36759c","error":"FailedCreateWorkerPod\npods \"fail-d7vcr-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:10.786Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"99b81460-6e96-4946-b964-4b927eff7927"}
{"level":"info","ts":"2026-06-05T09:41:10.786Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"99b81460-6e96-4946-b964-4b927eff7927"}
{"level":"info","ts":"2026-06-05T09:41:10.786Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"99b81460-6e96-4946-b964-4b927eff7927","1 head service found":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:10.786Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"99b81460-6e96-4946-b964-4b927eff7927","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-06-05T09:41:10.786Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"99b81460-6e96-4946-b964-4b927eff7927","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:41:10.786Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"99b81460-6e96-4946-b964-4b927eff7927","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:10.786Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"99b81460-6e96-4946-b964-4b927eff7927","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-05T09:41:10.786Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"99b81460-6e96-4946-b964-4b927eff7927","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:10.786Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"99b81460-6e96-4946-b964-4b927eff7927","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-05T09:41:10.786Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"99b81460-6e96-4946-b964-4b927eff7927"}
{"level":"info","ts":"2026-06-05T09:41:10.786Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"99b81460-6e96-4946-b964-4b927eff7927","nodeType":"worker","rayStartParams":{"address":"fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-05T09:41:10.786Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"99b81460-6e96-4946-b964-4b927eff7927","rayStartCmd":"ray start  --address=fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:41:10.786Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"99b81460-6e96-4946-b964-4b927eff7927","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:41:10.786Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"99b81460-6e96-4946-b964-4b927eff7927","enabled":true}
{"level":"error","ts":"2026-06-05T09:41:10.809Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"99b81460-6e96-4946-b964-4b927eff7927","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"fail-d7vcr-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:10.809Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"99b81460-6e96-4946-b964-4b927eff7927","error":"FailedCreateWorkerPod\npods \"fail-d7vcr-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated"}
{"level":"info","ts":"2026-06-05T09:41:10.809Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"99b81460-6e96-4946-b964-4b927eff7927"}
{"level":"error","ts":"2026-06-05T09:41:10.809Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"99b81460-6e96-4946-b964-4b927eff7927","error":"FailedCreateWorkerPod\npods \"fail-d7vcr-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:10.890Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"cd91140b-1c32-4faa-aab4-6ccc95cb03fc"}
{"level":"info","ts":"2026-06-05T09:41:10.890Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"cd91140b-1c32-4faa-aab4-6ccc95cb03fc"}
{"level":"info","ts":"2026-06-05T09:41:10.890Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"cd91140b-1c32-4faa-aab4-6ccc95cb03fc","1 head service found":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:10.890Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"cd91140b-1c32-4faa-aab4-6ccc95cb03fc","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-06-05T09:41:10.890Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"cd91140b-1c32-4faa-aab4-6ccc95cb03fc","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:41:10.890Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"cd91140b-1c32-4faa-aab4-6ccc95cb03fc","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:10.890Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"cd91140b-1c32-4faa-aab4-6ccc95cb03fc","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-05T09:41:10.890Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"cd91140b-1c32-4faa-aab4-6ccc95cb03fc","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:10.890Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"cd91140b-1c32-4faa-aab4-6ccc95cb03fc","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-05T09:41:10.890Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"cd91140b-1c32-4faa-aab4-6ccc95cb03fc"}
{"level":"info","ts":"2026-06-05T09:41:10.890Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"cd91140b-1c32-4faa-aab4-6ccc95cb03fc","nodeType":"worker","rayStartParams":{"address":"fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-05T09:41:10.890Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"cd91140b-1c32-4faa-aab4-6ccc95cb03fc","rayStartCmd":"ray start  --address=fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:41:10.890Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"cd91140b-1c32-4faa-aab4-6ccc95cb03fc","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:41:10.890Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"cd91140b-1c32-4faa-aab4-6ccc95cb03fc","enabled":true}
{"level":"error","ts":"2026-06-05T09:41:10.894Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"cd91140b-1c32-4faa-aab4-6ccc95cb03fc","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"fail-d7vcr-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:10.894Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"cd91140b-1c32-4faa-aab4-6ccc95cb03fc","error":"FailedCreateWorkerPod\npods \"fail-d7vcr-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated"}
{"level":"info","ts":"2026-06-05T09:41:10.894Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"cd91140b-1c32-4faa-aab4-6ccc95cb03fc"}
{"level":"error","ts":"2026-06-05T09:41:10.895Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"cd91140b-1c32-4faa-aab4-6ccc95cb03fc","error":"FailedCreateWorkerPod\npods \"fail-d7vcr-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:11.055Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"a133b941-e5e7-4635-a363-b9ae49f72c91"}
{"level":"info","ts":"2026-06-05T09:41:11.055Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"a133b941-e5e7-4635-a363-b9ae49f72c91"}
{"level":"info","ts":"2026-06-05T09:41:11.055Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"a133b941-e5e7-4635-a363-b9ae49f72c91","1 head service found":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:11.055Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"a133b941-e5e7-4635-a363-b9ae49f72c91","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-06-05T09:41:11.055Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"a133b941-e5e7-4635-a363-b9ae49f72c91","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:41:11.055Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"a133b941-e5e7-4635-a363-b9ae49f72c91","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:11.055Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"a133b941-e5e7-4635-a363-b9ae49f72c91","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-05T09:41:11.055Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"a133b941-e5e7-4635-a363-b9ae49f72c91","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:11.055Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"a133b941-e5e7-4635-a363-b9ae49f72c91","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-05T09:41:11.055Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"a133b941-e5e7-4635-a363-b9ae49f72c91"}
{"level":"info","ts":"2026-06-05T09:41:11.055Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"a133b941-e5e7-4635-a363-b9ae49f72c91","nodeType":"worker","rayStartParams":{"address":"fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-05T09:41:11.055Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"a133b941-e5e7-4635-a363-b9ae49f72c91","rayStartCmd":"ray start  --address=fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:41:11.055Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"a133b941-e5e7-4635-a363-b9ae49f72c91","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:41:11.055Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"a133b941-e5e7-4635-a363-b9ae49f72c91","enabled":true}
{"level":"error","ts":"2026-06-05T09:41:11.059Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"a133b941-e5e7-4635-a363-b9ae49f72c91","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"fail-d7vcr-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:11.059Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"a133b941-e5e7-4635-a363-b9ae49f72c91","error":"FailedCreateWorkerPod\npods \"fail-d7vcr-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated"}
{"level":"info","ts":"2026-06-05T09:41:11.059Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"a133b941-e5e7-4635-a363-b9ae49f72c91"}
{"level":"error","ts":"2026-06-05T09:41:11.059Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"a133b941-e5e7-4635-a363-b9ae49f72c91","error":"FailedCreateWorkerPod\npods \"fail-d7vcr-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:11.160Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"6aaecf6f-f6ea-4f20-88c2-633feb4387f4"}
{"level":"info","ts":"2026-06-05T09:41:11.160Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"6aaecf6f-f6ea-4f20-88c2-633feb4387f4"}
{"level":"info","ts":"2026-06-05T09:41:11.160Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"6aaecf6f-f6ea-4f20-88c2-633feb4387f4","1 head service found":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:11.161Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"6aaecf6f-f6ea-4f20-88c2-633feb4387f4","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-06-05T09:41:11.161Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"6aaecf6f-f6ea-4f20-88c2-633feb4387f4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:41:11.161Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"6aaecf6f-f6ea-4f20-88c2-633feb4387f4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:11.161Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"6aaecf6f-f6ea-4f20-88c2-633feb4387f4","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-05T09:41:11.161Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"6aaecf6f-f6ea-4f20-88c2-633feb4387f4","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:11.161Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"6aaecf6f-f6ea-4f20-88c2-633feb4387f4","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-05T09:41:11.161Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"6aaecf6f-f6ea-4f20-88c2-633feb4387f4"}
{"level":"info","ts":"2026-06-05T09:41:11.161Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"6aaecf6f-f6ea-4f20-88c2-633feb4387f4","nodeType":"worker","rayStartParams":{"address":"fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-05T09:41:11.161Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"6aaecf6f-f6ea-4f20-88c2-633feb4387f4","rayStartCmd":"ray start  --address=fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:41:11.161Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"6aaecf6f-f6ea-4f20-88c2-633feb4387f4","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:41:11.161Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"6aaecf6f-f6ea-4f20-88c2-633feb4387f4","enabled":true}
{"level":"error","ts":"2026-06-05T09:41:11.165Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"6aaecf6f-f6ea-4f20-88c2-633feb4387f4","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"fail-d7vcr-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:11.165Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"6aaecf6f-f6ea-4f20-88c2-633feb4387f4","error":"FailedCreateWorkerPod\npods \"fail-d7vcr-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated"}
{"level":"info","ts":"2026-06-05T09:41:11.165Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"6aaecf6f-f6ea-4f20-88c2-633feb4387f4","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"fail-d7vcr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:11.165Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"6aaecf6f-f6ea-4f20-88c2-633feb4387f4","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"fail-d7vcr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:11.165Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"6aaecf6f-f6ea-4f20-88c2-633feb4387f4","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"fail-d7vcr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:11.165Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"6aaecf6f-f6ea-4f20-88c2-633feb4387f4","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podIP":"10.132.0.16","serviceIP":"10.132.0.16","podName":"fail-d7vcr-head-8br8s","serviceName":"fail-d7vcr-head-svc"},"newHead":{"serviceName":"fail-d7vcr-head-svc"}}
{"level":"info","ts":"2026-06-05T09:41:11.165Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"6aaecf6f-f6ea-4f20-88c2-633feb4387f4","name":"fail-d7vcr","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:41:10Z","stateTransitionTimes":{"ready":"2026-06-05T09:39:55Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.16","serviceIP":"10.132.0.16","podName":"fail-d7vcr-head-8br8s","serviceName":"fail-d7vcr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:10Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:39:55Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-05T09:41:10Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"fail-d7vcr-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:41:11Z","stateTransitionTimes":{"ready":"2026-06-05T09:39:55Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-d7vcr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:10Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:39:55Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-05T09:41:10Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"fail-d7vcr-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:41:11.177Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"6aaecf6f-f6ea-4f20-88c2-633feb4387f4"}
{"level":"error","ts":"2026-06-05T09:41:11.177Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"6aaecf6f-f6ea-4f20-88c2-633feb4387f4","error":"FailedCreateWorkerPod\npods \"fail-d7vcr-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:11.177Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"efe3c268-0672-4be4-be09-e9e0392c5012","rayCluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:41:11.178Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"1a111dc1-c0e8-41ed-8a85-c8b594715bdc","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:41:11.178Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"1a111dc1-c0e8-41ed-8a85-c8b594715bdc","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-05 09:40:00 +0000 UTC","Now":"2026-06-05T09:41:11.178Z","ShutdownTime":"2026-06-05T09:40:00.000Z"}
{"level":"info","ts":"2026-06-05T09:41:11.177Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"8602c467-41fc-44bf-b56b-11b312048991"}
{"level":"info","ts":"2026-06-05T09:41:11.178Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"8602c467-41fc-44bf-b56b-11b312048991"}
{"level":"info","ts":"2026-06-05T09:41:11.177Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"710b58e8-8ce5-433e-a1e8-f328b8c56647","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:41:11.178Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"8602c467-41fc-44bf-b56b-11b312048991","1 head service found":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:11.178Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"8602c467-41fc-44bf-b56b-11b312048991"}
{"level":"info","ts":"2026-06-05T09:41:11.178Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"8602c467-41fc-44bf-b56b-11b312048991","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:41:11.178Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"8602c467-41fc-44bf-b56b-11b312048991"}
{"level":"info","ts":"2026-06-05T09:41:11.178Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"8602c467-41fc-44bf-b56b-11b312048991","mode":"IntegratedOAuth","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:41:11.178Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"8602c467-41fc-44bf-b56b-11b312048991","cluster":"fail-d7vcr","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:41:11.178Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"8602c467-41fc-44bf-b56b-11b312048991","cluster":"fail-d7vcr","authType":"IntegratedOAuth","serviceAccount":"fail-d7vcr-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-05T09:41:11.178Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"8602c467-41fc-44bf-b56b-11b312048991","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"fail-d7vcr","ray.io/group":"headgroup","ray.io/identifier":"fail-d7vcr-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-05T09:41:11.178Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"8602c467-41fc-44bf-b56b-11b312048991","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-05T09:41:11.178Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"8602c467-41fc-44bf-b56b-11b312048991","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-05T09:41:11.178Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"8602c467-41fc-44bf-b56b-11b312048991","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-05T09:41:11.178Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"8602c467-41fc-44bf-b56b-11b312048991","enabled":true}
{"level":"error","ts":"2026-06-05T09:41:11.181Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"8602c467-41fc-44bf-b56b-11b312048991","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"fail-d7vcr-head-\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:11.181Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"8602c467-41fc-44bf-b56b-11b312048991","error":"FailedCreateHeadPod\npods \"fail-d7vcr-head-\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated"}
{"level":"info","ts":"2026-06-05T09:41:11.181Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"8602c467-41fc-44bf-b56b-11b312048991","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"fail-d7vcr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:11.181Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"8602c467-41fc-44bf-b56b-11b312048991","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"fail-d7vcr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:11.181Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"8602c467-41fc-44bf-b56b-11b312048991","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"fail-d7vcr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:11.181Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"8602c467-41fc-44bf-b56b-11b312048991","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:10Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:39:55Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-05T09:41:10Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"fail-d7vcr-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:10Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:39:55Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-05T09:41:10Z","reason":"FailedCreateHeadPod","message":"FailedCreateHeadPod\npods \"fail-d7vcr-head-\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated"}]}
{"level":"info","ts":"2026-06-05T09:41:11.181Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"8602c467-41fc-44bf-b56b-11b312048991","name":"fail-d7vcr","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:41:11Z","stateTransitionTimes":{"ready":"2026-06-05T09:39:55Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-d7vcr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:10Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:39:55Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-05T09:41:10Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"fail-d7vcr-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:41:11Z","stateTransitionTimes":{"ready":"2026-06-05T09:39:55Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-d7vcr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:10Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:39:55Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:39:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:39:30Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-05T09:41:10Z","reason":"FailedCreateHeadPod","message":"FailedCreateHeadPod\npods \"fail-d7vcr-head-\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:41:11.183Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"710b58e8-8ce5-433e-a1e8-f328b8c56647","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:41:11.206Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"efe3c268-0672-4be4-be09-e9e0392c5012","certificate":"ray-head-cert-fail-d7vcr","dnsNames":["fail-d7vcr-head-svc","localhost","fail-d7vcr-head-svc.test-ns-l4cdh.svc","fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local"],"ipAddresses":["127.0.0.1"]}
{"level":"info","ts":"2026-06-05T09:41:11.206Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"fd911612-c0dd-40c2-90e0-e112a0e3040e","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:41:11.206Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"8602c467-41fc-44bf-b56b-11b312048991"}
{"level":"error","ts":"2026-06-05T09:41:11.206Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"8602c467-41fc-44bf-b56b-11b312048991","error":"FailedCreateHeadPod\npods \"fail-d7vcr-head-\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:11.206Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"21d17708-7470-4239-941f-6a964b130173","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:41:11.206Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"21d17708-7470-4239-941f-6a964b130173","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-05 09:40:00 +0000 UTC","Now":"2026-06-05T09:41:11.206Z","ShutdownTime":"2026-06-05T09:40:00.000Z"}
{"level":"info","ts":"2026-06-05T09:41:11.211Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"fd911612-c0dd-40c2-90e0-e112a0e3040e","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:41:11.219Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"efe3c268-0672-4be4-be09-e9e0392c5012","certificate":"ray-worker-cert-fail-d7vcr","dnsNames":["fail-d7vcr-worker-svc","localhost","fail-d7vcr-small-group","fail-d7vcr-worker-svc.test-ns-l4cdh.svc","fail-d7vcr-worker-svc.test-ns-l4cdh.svc.cluster.local","*.fail-d7vcr-worker-svc.test-ns-l4cdh.svc","*.fail-d7vcr-worker-svc.test-ns-l4cdh.svc.cluster.local","*-worker-*.test-ns-l4cdh.svc","*-worker-*.test-ns-l4cdh.svc.cluster.local"],"ipAddresses":["127.0.0.1"]}
{"level":"info","ts":"2026-06-05T09:41:11.219Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"efe3c268-0672-4be4-be09-e9e0392c5012"}
{"level":"info","ts":"2026-06-05T09:41:11.219Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"c123bbef-4ba0-4770-91ac-48c18d9f57d5","rayCluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:41:11.219Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"c123bbef-4ba0-4770-91ac-48c18d9f57d5"}
{"level":"info","ts":"2026-06-05T09:41:11.379Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"08b3911c-c8a6-4c41-9afd-0c10735dacb5"}
{"level":"info","ts":"2026-06-05T09:41:11.379Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"08b3911c-c8a6-4c41-9afd-0c10735dacb5"}
{"level":"info","ts":"2026-06-05T09:41:11.379Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"08b3911c-c8a6-4c41-9afd-0c10735dacb5","1 head service found":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:11.379Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"08b3911c-c8a6-4c41-9afd-0c10735dacb5"}
{"level":"info","ts":"2026-06-05T09:41:11.379Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"08b3911c-c8a6-4c41-9afd-0c10735dacb5","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:41:11.379Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"08b3911c-c8a6-4c41-9afd-0c10735dacb5"}
{"level":"info","ts":"2026-06-05T09:41:11.379Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"08b3911c-c8a6-4c41-9afd-0c10735dacb5","mode":"IntegratedOAuth","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:41:11.379Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"08b3911c-c8a6-4c41-9afd-0c10735dacb5","cluster":"fail-d7vcr","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:41:11.380Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"08b3911c-c8a6-4c41-9afd-0c10735dacb5","cluster":"fail-d7vcr","authType":"IntegratedOAuth","serviceAccount":"fail-d7vcr-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-05T09:41:11.380Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"08b3911c-c8a6-4c41-9afd-0c10735dacb5","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"fail-d7vcr","ray.io/group":"headgroup","ray.io/identifier":"fail-d7vcr-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-05T09:41:11.380Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"08b3911c-c8a6-4c41-9afd-0c10735dacb5","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-05T09:41:11.380Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"08b3911c-c8a6-4c41-9afd-0c10735dacb5","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-05T09:41:11.380Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"08b3911c-c8a6-4c41-9afd-0c10735dacb5","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-05T09:41:11.380Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"08b3911c-c8a6-4c41-9afd-0c10735dacb5","enabled":true}
{"level":"error","ts":"2026-06-05T09:41:11.384Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"08b3911c-c8a6-4c41-9afd-0c10735dacb5","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"fail-d7vcr-head-\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:11.384Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"08b3911c-c8a6-4c41-9afd-0c10735dacb5","error":"FailedCreateHeadPod\npods \"fail-d7vcr-head-\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated"}
{"level":"info","ts":"2026-06-05T09:41:11.384Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"08b3911c-c8a6-4c41-9afd-0c10735dacb5","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"fail-d7vcr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:11.384Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"08b3911c-c8a6-4c41-9afd-0c10735dacb5","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"fail-d7vcr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:11.384Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"08b3911c-c8a6-4c41-9afd-0c10735dacb5","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"fail-d7vcr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:11.384Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"08b3911c-c8a6-4c41-9afd-0c10735dacb5"}
{"level":"error","ts":"2026-06-05T09:41:11.384Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"08b3911c-c8a6-4c41-9afd-0c10735dacb5","error":"FailedCreateHeadPod\npods \"fail-d7vcr-head-\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:13.700Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"daa333de-9b5b-4387-bdbd-820c73173142"}
{"level":"info","ts":"2026-06-05T09:41:13.700Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"daa333de-9b5b-4387-bdbd-820c73173142"}
{"level":"info","ts":"2026-06-05T09:41:13.700Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"daa333de-9b5b-4387-bdbd-820c73173142","1 head service found":"long-running-qkmps-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:13.701Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"daa333de-9b5b-4387-bdbd-820c73173142","Found 1 head Pod":"long-running-qkmps-head-r2zkr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:41:13.701Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"daa333de-9b5b-4387-bdbd-820c73173142","head Pod":"long-running-qkmps-head-r2zkr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qkmps-head-r2zkr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:13.701Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"daa333de-9b5b-4387-bdbd-820c73173142","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:41:13.701Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"daa333de-9b5b-4387-bdbd-820c73173142","worker Pod":"long-running-qkmps-small-group-worker-pvt7t","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qkmps-small-group-worker-pvt7t. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:13.701Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"daa333de-9b5b-4387-bdbd-820c73173142","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:13.701Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"daa333de-9b5b-4387-bdbd-820c73173142","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:41:13.701Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"daa333de-9b5b-4387-bdbd-820c73173142","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:13.701Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"daa333de-9b5b-4387-bdbd-820c73173142","podName":"long-running-qkmps-head-r2zkr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:41:13.701Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"daa333de-9b5b-4387-bdbd-820c73173142","podName":"long-running-qkmps-head-r2zkr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:41:13.701Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"daa333de-9b5b-4387-bdbd-820c73173142","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:41:13.701Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"daa333de-9b5b-4387-bdbd-820c73173142","seconds":300}
{"level":"info","ts":"2026-06-05T09:41:13.735Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"90e5b0d6-0733-4ce5-a67b-e1895d316a4c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:13.735Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"90e5b0d6-0733-4ce5-a67b-e1895d316a4c","name":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:13.735Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"90e5b0d6-0733-4ce5-a67b-e1895d316a4c","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:13.735Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"90e5b0d6-0733-4ce5-a67b-e1895d316a4c"}
{"level":"info","ts":"2026-06-05T09:41:13.735Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"90e5b0d6-0733-4ce5-a67b-e1895d316a4c","RayCluster":"long-running-qkmps","State":""}
{"level":"info","ts":"2026-06-05T09:41:13.944Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c4da69bd-39e6-43ae-b1f6-a6cdbd39f959"}
{"level":"info","ts":"2026-06-05T09:41:13.944Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c4da69bd-39e6-43ae-b1f6-a6cdbd39f959"}
{"level":"info","ts":"2026-06-05T09:41:13.944Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c4da69bd-39e6-43ae-b1f6-a6cdbd39f959","1 head service found":"fail-d7vcr-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:13.944Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c4da69bd-39e6-43ae-b1f6-a6cdbd39f959"}
{"level":"info","ts":"2026-06-05T09:41:13.944Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c4da69bd-39e6-43ae-b1f6-a6cdbd39f959","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:41:13.944Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c4da69bd-39e6-43ae-b1f6-a6cdbd39f959"}
{"level":"info","ts":"2026-06-05T09:41:13.944Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c4da69bd-39e6-43ae-b1f6-a6cdbd39f959","mode":"IntegratedOAuth","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:41:13.944Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c4da69bd-39e6-43ae-b1f6-a6cdbd39f959","cluster":"fail-d7vcr","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:41:13.944Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c4da69bd-39e6-43ae-b1f6-a6cdbd39f959","cluster":"fail-d7vcr","authType":"IntegratedOAuth","serviceAccount":"fail-d7vcr-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-05T09:41:13.944Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c4da69bd-39e6-43ae-b1f6-a6cdbd39f959","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"fail-d7vcr","ray.io/group":"headgroup","ray.io/identifier":"fail-d7vcr-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-05T09:41:13.944Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c4da69bd-39e6-43ae-b1f6-a6cdbd39f959","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-05T09:41:13.944Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c4da69bd-39e6-43ae-b1f6-a6cdbd39f959","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-05T09:41:13.944Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c4da69bd-39e6-43ae-b1f6-a6cdbd39f959","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-d7vcr-head-svc.test-ns-l4cdh.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-05T09:41:13.944Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c4da69bd-39e6-43ae-b1f6-a6cdbd39f959","enabled":true}
{"level":"error","ts":"2026-06-05T09:41:13.949Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c4da69bd-39e6-43ae-b1f6-a6cdbd39f959","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"fail-d7vcr-head-\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:13.949Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c4da69bd-39e6-43ae-b1f6-a6cdbd39f959","error":"FailedCreateHeadPod\npods \"fail-d7vcr-head-\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated"}
{"level":"info","ts":"2026-06-05T09:41:13.949Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c4da69bd-39e6-43ae-b1f6-a6cdbd39f959","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"fail-d7vcr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:13.949Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c4da69bd-39e6-43ae-b1f6-a6cdbd39f959","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"fail-d7vcr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:13.949Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c4da69bd-39e6-43ae-b1f6-a6cdbd39f959","filter labels":["test-ns-l4cdh",{"ray.io/cluster":"fail-d7vcr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:13.949Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c4da69bd-39e6-43ae-b1f6-a6cdbd39f959"}
{"level":"error","ts":"2026-06-05T09:41:13.949Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"c4da69bd-39e6-43ae-b1f6-a6cdbd39f959","error":"FailedCreateHeadPod\npods \"fail-d7vcr-head-\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:14.714Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"9c226637-4ad2-47d3-9717-92ed543f80a1"}
{"level":"info","ts":"2026-06-05T09:41:14.714Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"9c226637-4ad2-47d3-9717-92ed543f80a1"}
{"level":"info","ts":"2026-06-05T09:41:14.714Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"9c226637-4ad2-47d3-9717-92ed543f80a1","1 head service found":"long-running-qkmps-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:14.714Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"9c226637-4ad2-47d3-9717-92ed543f80a1","Found 1 head Pod":"long-running-qkmps-head-r2zkr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:41:14.714Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"9c226637-4ad2-47d3-9717-92ed543f80a1","head Pod":"long-running-qkmps-head-r2zkr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qkmps-head-r2zkr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:14.714Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"9c226637-4ad2-47d3-9717-92ed543f80a1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:41:14.714Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"9c226637-4ad2-47d3-9717-92ed543f80a1","worker Pod":"long-running-qkmps-small-group-worker-pvt7t","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qkmps-small-group-worker-pvt7t. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:14.714Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"9c226637-4ad2-47d3-9717-92ed543f80a1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:14.714Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"9c226637-4ad2-47d3-9717-92ed543f80a1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:41:14.714Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"9c226637-4ad2-47d3-9717-92ed543f80a1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:14.714Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"9c226637-4ad2-47d3-9717-92ed543f80a1","podName":"long-running-qkmps-head-r2zkr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:41:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:41:14.715Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"9c226637-4ad2-47d3-9717-92ed543f80a1","podName":"long-running-qkmps-head-r2zkr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:41:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:41:14.715Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"9c226637-4ad2-47d3-9717-92ed543f80a1","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:41:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:41:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-05T09:41:14.715Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"9c226637-4ad2-47d3-9717-92ed543f80a1","name":"long-running-qkmps","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:41:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"long-running-qkmps-head-r2zkr","serviceName":"long-running-qkmps-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:41:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:41:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"long-running-qkmps-head-r2zkr","serviceName":"long-running-qkmps-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:41:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:41:14.728Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"33e10d7c-724f-4f93-b730-bc457984e452","cluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:14.728Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"0ea2bfff-f1f6-4796-a9a9-05e5e6fdde6d","rayCluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:14.728Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"427d8dce-002a-446f-a60d-301d658c5007","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:14.728Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"427d8dce-002a-446f-a60d-301d658c5007","name":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:14.728Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"427d8dce-002a-446f-a60d-301d658c5007","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:14.728Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"0ea2bfff-f1f6-4796-a9a9-05e5e6fdde6d","pod":"long-running-qkmps-head-r2zkr","ip":"10.133.0.38"}
{"level":"info","ts":"2026-06-05T09:41:14.728Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"0ea2bfff-f1f6-4796-a9a9-05e5e6fdde6d","pod":"long-running-qkmps-small-group-worker-pvt7t","ip":"10.132.0.18"}
{"level":"info","ts":"2026-06-05T09:41:14.728Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"427d8dce-002a-446f-a60d-301d658c5007"}
{"level":"info","ts":"2026-06-05T09:41:14.728Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"427d8dce-002a-446f-a60d-301d658c5007","RayCluster":"long-running-qkmps","State":""}
{"level":"info","ts":"2026-06-05T09:41:14.728Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"0ea2bfff-f1f6-4796-a9a9-05e5e6fdde6d","rayCluster":"long-running-qkmps","podIPs":["10.133.0.38","10.132.0.18"]}
{"level":"info","ts":"2026-06-05T09:41:14.750Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"33e10d7c-724f-4f93-b730-bc457984e452","name":"long-running-qkmps-head"}
{"level":"info","ts":"2026-06-05T09:41:14.810Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"33e10d7c-724f-4f93-b730-bc457984e452","name":"long-running-qkmps-workers"}
{"level":"info","ts":"2026-06-05T09:41:14.810Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"33e10d7c-724f-4f93-b730-bc457984e452","cluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:15.271Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"5cbb015a-9572-44c3-b0eb-da75c6cf391a","rayCluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:41:15.271Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"c7d92765-8016-4ac0-9b22-d5225adf9422","namespacedName":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:41:15.271Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"5cbb015a-9572-44c3-b0eb-da75c6cf391a","rayCluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:41:15.271Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"612d4974-2629-452e-b577-147d3af00d04"}
{"level":"info","ts":"2026-06-05T09:41:15.271Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"c7d92765-8016-4ac0-9b22-d5225adf9422","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:41:15.271Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"c7d92765-8016-4ac0-9b22-d5225adf9422","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:41:15.271Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"reconcileID":"5b1e3bb5-2a59-4bab-b76d-10693d7d581a"}
{"level":"info","ts":"2026-06-05T09:41:15.271Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"c7d92765-8016-4ac0-9b22-d5225adf9422","namespace":"test-ns-l4cdh","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-05T09:41:15.271Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"17a6c929-9a09-46c3-9560-e50863b1c14a","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:41:15.271Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"17a6c929-9a09-46c3-9560-e50863b1c14a","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-05 09:40:00 +0000 UTC","Now":"2026-06-05T09:41:15.271Z","ShutdownTime":"2026-06-05T09:40:00.000Z"}
{"level":"error","ts":"2026-06-05T09:41:15.274Z","msg":"Failed to create CA Certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"5cbb015a-9572-44c3-b0eb-da75c6cf391a","error":"certificates.cert-manager.io \"ray-ca-certificate-fail-d7vcr\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createCACertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:890\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileRootCA\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:483\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:165\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-05T09:41:15.274Z","msg":"Failed to reconcile RootCA","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"5cbb015a-9572-44c3-b0eb-da75c6cf391a","error":"certificates.cert-manager.io \"ray-ca-certificate-fail-d7vcr\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:166\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:15.274Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"5cbb015a-9572-44c3-b0eb-da75c6cf391a"}
{"level":"error","ts":"2026-06-05T09:41:15.274Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"5cbb015a-9572-44c3-b0eb-da75c6cf391a","error":"certificates.cert-manager.io \"ray-ca-certificate-fail-d7vcr\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:15.280Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"037ba68e-b20c-4ce2-b114-4c31a384816b","rayCluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:41:15.280Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"037ba68e-b20c-4ce2-b114-4c31a384816b","rayCluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:41:15.281Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"c7d92765-8016-4ac0-9b22-d5225adf9422","name":"kuberay-gateway-access","namespace":"test-ns-l4cdh"}
{"level":"info","ts":"2026-06-05T09:41:15.282Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"64e2f6a2-bd39-46bb-b806-2e4d7155ee9e"}
{"level":"error","ts":"2026-06-05T09:41:15.283Z","msg":"Failed to create CA Certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"037ba68e-b20c-4ce2-b114-4c31a384816b","error":"certificates.cert-manager.io \"ray-ca-certificate-fail-d7vcr\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createCACertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:890\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileRootCA\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:483\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:165\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-05T09:41:15.283Z","msg":"Failed to reconcile RootCA","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"037ba68e-b20c-4ce2-b114-4c31a384816b","error":"certificates.cert-manager.io \"ray-ca-certificate-fail-d7vcr\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:166\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:15.283Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"037ba68e-b20c-4ce2-b114-4c31a384816b"}
{"level":"error","ts":"2026-06-05T09:41:15.283Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"037ba68e-b20c-4ce2-b114-4c31a384816b","error":"certificates.cert-manager.io \"ray-ca-certificate-fail-d7vcr\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:15.285Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"c72e1c36-b48f-42e3-8608-2adac8350008"}
{"level":"info","ts":"2026-06-05T09:41:15.289Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"c7d92765-8016-4ac0-9b22-d5225adf9422","name":"test-ns-l4cdh-fail-d7vcr","namespace":"opendatahub"}
{"level":"info","ts":"2026-06-05T09:41:15.294Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"8038f775-162d-4bf0-9f56-59549c70af4c","rayCluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:41:15.294Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"8038f775-162d-4bf0-9f56-59549c70af4c","rayCluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:41:15.294Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"c7d92765-8016-4ac0-9b22-d5225adf9422","configMap":"kube-rbac-proxy-config-fail-d7vcr"}
{"level":"error","ts":"2026-06-05T09:41:15.297Z","msg":"Failed to create CA Certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"8038f775-162d-4bf0-9f56-59549c70af4c","error":"certificates.cert-manager.io \"ray-ca-certificate-fail-d7vcr\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createCACertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:890\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileRootCA\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:483\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:165\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-05T09:41:15.297Z","msg":"Failed to reconcile RootCA","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"8038f775-162d-4bf0-9f56-59549c70af4c","error":"certificates.cert-manager.io \"ray-ca-certificate-fail-d7vcr\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:166\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:15.297Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"8038f775-162d-4bf0-9f56-59549c70af4c"}
{"level":"error","ts":"2026-06-05T09:41:15.297Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"8038f775-162d-4bf0-9f56-59549c70af4c","error":"certificates.cert-manager.io \"ray-ca-certificate-fail-d7vcr\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:15.300Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"c7d92765-8016-4ac0-9b22-d5225adf9422","serviceAccount":"fail-d7vcr-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:41:15.307Z","logger":"raycluster-resource","msg":"default","name":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:41:15.307Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-d7vcr","namespace":"test-ns-l4cdh"}
{"level":"info","ts":"2026-06-05T09:41:15.317Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"11aba8c1-8cf8-4b28-b2c4-2dcf84e17fe4","rayCluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:41:15.317Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"11aba8c1-8cf8-4b28-b2c4-2dcf84e17fe4","rayCluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:41:15.318Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"c7d92765-8016-4ac0-9b22-d5225adf9422","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:41:15.318Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"89accb72-4044-49dd-bef3-3171f47aba07","namespacedName":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:41:15.318Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"89accb72-4044-49dd-bef3-3171f47aba07","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:41:15.318Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"89accb72-4044-49dd-bef3-3171f47aba07","cluster":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:41:15.318Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"89accb72-4044-49dd-bef3-3171f47aba07","namespace":"test-ns-l4cdh","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-05T09:41:15.318Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"89accb72-4044-49dd-bef3-3171f47aba07","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-06-05T09:41:15.318Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"0adf319d-8c83-436c-9f26-5fb2502d5750"}
{"level":"info","ts":"2026-06-05T09:41:15.318Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"7f573414-22e2-4375-9a30-eb1345473bb8","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-05T09:41:15.318Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"7f573414-22e2-4375-9a30-eb1345473bb8","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-05 09:40:00 +0000 UTC","Now":"2026-06-05T09:41:15.318Z","ShutdownTime":"2026-06-05T09:40:00.000Z"}
{"level":"error","ts":"2026-06-05T09:41:15.320Z","msg":"Failed to create CA Certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"11aba8c1-8cf8-4b28-b2c4-2dcf84e17fe4","error":"certificates.cert-manager.io \"ray-ca-certificate-fail-d7vcr\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createCACertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:890\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileRootCA\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:483\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:165\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-05T09:41:15.320Z","msg":"Failed to reconcile RootCA","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"11aba8c1-8cf8-4b28-b2c4-2dcf84e17fe4","error":"certificates.cert-manager.io \"ray-ca-certificate-fail-d7vcr\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:166\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:15.320Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"11aba8c1-8cf8-4b28-b2c4-2dcf84e17fe4"}
{"level":"error","ts":"2026-06-05T09:41:15.320Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"11aba8c1-8cf8-4b28-b2c4-2dcf84e17fe4","error":"certificates.cert-manager.io \"ray-ca-certificate-fail-d7vcr\" is forbidden: unable to create new content in namespace test-ns-l4cdh because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:15.320Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"a9c536ed-181a-40c0-873b-0729b94cf684","namespace":"test-ns-l4cdh","clusterName":"fail-d7vcr"}
{"level":"error","ts":"2026-06-05T09:41:15.323Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"89accb72-4044-49dd-bef3-3171f47aba07","error":"rayclusters.ray.io \"fail-d7vcr\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:214\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:155\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:15.323Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"89accb72-4044-49dd-bef3-3171f47aba07"}
{"level":"error","ts":"2026-06-05T09:41:15.323Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"89accb72-4044-49dd-bef3-3171f47aba07","error":"rayclusters.ray.io \"fail-d7vcr\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:15.323Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"799beda3-82ea-4814-babe-4043e8755e97","namespacedName":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:41:15.323Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"799beda3-82ea-4814-babe-4043e8755e97"}
{"level":"info","ts":"2026-06-05T09:41:15.323Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"799beda3-82ea-4814-babe-4043e8755e97","namespace":"test-ns-l4cdh","authClusters":0}
{"level":"info","ts":"2026-06-05T09:41:15.323Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"799beda3-82ea-4814-babe-4043e8755e97","namespace":"test-ns-l4cdh"}
{"level":"info","ts":"2026-06-05T09:41:15.325Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"86407ba9-2805-4d7f-bc12-685f90ada606","namespacedName":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:41:15.325Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"86407ba9-2805-4d7f-bc12-685f90ada606"}
{"level":"info","ts":"2026-06-05T09:41:15.325Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"86407ba9-2805-4d7f-bc12-685f90ada606","namespace":"test-ns-l4cdh","authClusters":0}
{"level":"info","ts":"2026-06-05T09:41:15.325Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"86407ba9-2805-4d7f-bc12-685f90ada606","namespace":"test-ns-l4cdh"}
{"level":"error","ts":"2026-06-05T09:41:15.326Z","msg":"Failed to delete CA issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"a9c536ed-181a-40c0-873b-0729b94cf684","issuer":"ray-ca-issuer-fail-d7vcr","error":"issuers.cert-manager.io \"ray-ca-issuer-fail-d7vcr\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:247\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:15.328Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"b837a7b8-0f00-40fb-b94e-dd7cb05c5e0d","namespacedName":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"}}
{"level":"info","ts":"2026-06-05T09:41:15.328Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"b837a7b8-0f00-40fb-b94e-dd7cb05c5e0d"}
{"level":"info","ts":"2026-06-05T09:41:15.328Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"b837a7b8-0f00-40fb-b94e-dd7cb05c5e0d","namespace":"test-ns-l4cdh","authClusters":0}
{"level":"info","ts":"2026-06-05T09:41:15.328Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"b837a7b8-0f00-40fb-b94e-dd7cb05c5e0d","namespace":"test-ns-l4cdh"}
{"level":"error","ts":"2026-06-05T09:41:15.329Z","msg":"Failed to delete SelfSigned issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"a9c536ed-181a-40c0-873b-0729b94cf684","issuer":"ray-selfsigned-issuer-fail-d7vcr","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-fail-d7vcr\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:254\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:15.329Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"a9c536ed-181a-40c0-873b-0729b94cf684","errorCount":2}
{"level":"info","ts":"2026-06-05T09:41:15.329Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"a9c536ed-181a-40c0-873b-0729b94cf684"}
{"level":"error","ts":"2026-06-05T09:41:15.329Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"a9c536ed-181a-40c0-873b-0729b94cf684","error":"issuers.cert-manager.io \"ray-ca-issuer-fail-d7vcr\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:15.361Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"9a04b69c-2cfc-4581-89f3-0e364cc3c7e1","namespace":"test-ns-l4cdh","clusterName":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:41:15.361Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-d7vcr","namespace":"test-ns-l4cdh"},"namespace":"test-ns-l4cdh","name":"fail-d7vcr","reconcileID":"9a04b69c-2cfc-4581-89f3-0e364cc3c7e1","namespace":"test-ns-l4cdh","clusterName":"fail-d7vcr"}
{"level":"info","ts":"2026-06-05T09:41:16.391Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"88ae371a-ec7e-439b-b9b0-c6bc3fa4dc19","DeletionTimestamp":"2026-06-05 09:41:16 +0000 UTC"}
{"level":"info","ts":"2026-06-05T09:41:16.391Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"88ae371a-ec7e-439b-b9b0-c6bc3fa4dc19","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-05T09:41:16.426Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"ac273561-7a9a-4d39-a85e-d801e4ac50ef","DeletionTimestamp":"2026-06-05 09:41:16 +0000 UTC"}
{"level":"info","ts":"2026-06-05T09:41:16.426Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"ac273561-7a9a-4d39-a85e-d801e4ac50ef","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-05T09:41:16.459Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"df325271-cf2f-4d3a-ab3b-af026b6e29b4"}
{"level":"info","ts":"2026-06-05T09:41:16.459Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"b434e2d4-a204-4c67-aa74-77af2f9f3129"}
{"level":"info","ts":"2026-06-05T09:41:16.729Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"308eced3-9f06-4869-9395-0dc03c035c6c"}
{"level":"info","ts":"2026-06-05T09:41:16.729Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"308eced3-9f06-4869-9395-0dc03c035c6c"}
{"level":"info","ts":"2026-06-05T09:41:16.729Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"308eced3-9f06-4869-9395-0dc03c035c6c","1 head service found":"long-running-qkmps-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:16.729Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"308eced3-9f06-4869-9395-0dc03c035c6c","Found 1 head Pod":"long-running-qkmps-head-r2zkr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:41:16.729Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"308eced3-9f06-4869-9395-0dc03c035c6c","head Pod":"long-running-qkmps-head-r2zkr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qkmps-head-r2zkr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:16.729Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"308eced3-9f06-4869-9395-0dc03c035c6c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:41:16.729Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"308eced3-9f06-4869-9395-0dc03c035c6c","worker Pod":"long-running-qkmps-small-group-worker-pvt7t","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qkmps-small-group-worker-pvt7t. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:16.729Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"308eced3-9f06-4869-9395-0dc03c035c6c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:16.729Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"308eced3-9f06-4869-9395-0dc03c035c6c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:41:16.729Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"308eced3-9f06-4869-9395-0dc03c035c6c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:16.729Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"308eced3-9f06-4869-9395-0dc03c035c6c","podName":"long-running-qkmps-head-r2zkr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:41:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:41:16.729Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"308eced3-9f06-4869-9395-0dc03c035c6c","podName":"long-running-qkmps-head-r2zkr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:41:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:41:16.729Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"308eced3-9f06-4869-9395-0dc03c035c6c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:41:16.729Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"308eced3-9f06-4869-9395-0dc03c035c6c","seconds":300}
{"level":"info","ts":"2026-06-05T09:41:16.736Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"e1458282-6fa7-4dc4-ad50-9e8396722b01","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:16.736Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"e1458282-6fa7-4dc4-ad50-9e8396722b01","name":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:16.736Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"e1458282-6fa7-4dc4-ad50-9e8396722b01","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:16.736Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"e1458282-6fa7-4dc4-ad50-9e8396722b01"}
{"level":"info","ts":"2026-06-05T09:41:16.736Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"e1458282-6fa7-4dc4-ad50-9e8396722b01","RayCluster":"long-running-qkmps","State":""}
{"level":"info","ts":"2026-06-05T09:41:18.188Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"6d97a5a1-e811-4f30-a6f7-4151c00cfa29"}
{"level":"info","ts":"2026-06-05T09:41:18.188Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"6d97a5a1-e811-4f30-a6f7-4151c00cfa29"}
{"level":"info","ts":"2026-06-05T09:41:18.188Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"6d97a5a1-e811-4f30-a6f7-4151c00cfa29","1 head service found":"long-running-qkmps-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:18.188Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"6d97a5a1-e811-4f30-a6f7-4151c00cfa29","Found 1 head Pod":"long-running-qkmps-head-r2zkr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:41:18.188Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"6d97a5a1-e811-4f30-a6f7-4151c00cfa29","head Pod":"long-running-qkmps-head-r2zkr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qkmps-head-r2zkr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:18.188Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"6d97a5a1-e811-4f30-a6f7-4151c00cfa29","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:41:18.188Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"6d97a5a1-e811-4f30-a6f7-4151c00cfa29","worker Pod":"long-running-qkmps-small-group-worker-pvt7t","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qkmps-small-group-worker-pvt7t. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:18.188Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"6d97a5a1-e811-4f30-a6f7-4151c00cfa29","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:18.188Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"6d97a5a1-e811-4f30-a6f7-4151c00cfa29","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:41:18.188Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"6d97a5a1-e811-4f30-a6f7-4151c00cfa29","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:18.188Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"6d97a5a1-e811-4f30-a6f7-4151c00cfa29","podName":"long-running-qkmps-head-r2zkr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:41:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:41:18.188Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"6d97a5a1-e811-4f30-a6f7-4151c00cfa29","podName":"long-running-qkmps-head-r2zkr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:41:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:41:18.188Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"6d97a5a1-e811-4f30-a6f7-4151c00cfa29","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:41:18.189Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"6d97a5a1-e811-4f30-a6f7-4151c00cfa29","seconds":300}
{"level":"info","ts":"2026-06-05T09:41:19.197Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"277cce42-6195-41e9-8bc1-b13ead6013c0"}
{"level":"info","ts":"2026-06-05T09:41:19.197Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"277cce42-6195-41e9-8bc1-b13ead6013c0"}
{"level":"info","ts":"2026-06-05T09:41:19.197Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"277cce42-6195-41e9-8bc1-b13ead6013c0","1 head service found":"long-running-qkmps-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:19.197Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"277cce42-6195-41e9-8bc1-b13ead6013c0","Found 1 head Pod":"long-running-qkmps-head-r2zkr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:41:19.197Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"277cce42-6195-41e9-8bc1-b13ead6013c0","head Pod":"long-running-qkmps-head-r2zkr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qkmps-head-r2zkr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:19.197Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"277cce42-6195-41e9-8bc1-b13ead6013c0","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:41:19.197Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"277cce42-6195-41e9-8bc1-b13ead6013c0","worker Pod":"long-running-qkmps-small-group-worker-pvt7t","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qkmps-small-group-worker-pvt7t. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:19.197Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"277cce42-6195-41e9-8bc1-b13ead6013c0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:19.197Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"277cce42-6195-41e9-8bc1-b13ead6013c0","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:41:19.197Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"277cce42-6195-41e9-8bc1-b13ead6013c0","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:19.197Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"277cce42-6195-41e9-8bc1-b13ead6013c0","podName":"long-running-qkmps-head-r2zkr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:41:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:41:19.197Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"277cce42-6195-41e9-8bc1-b13ead6013c0","podName":"long-running-qkmps-head-r2zkr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:41:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:41:19.197Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"277cce42-6195-41e9-8bc1-b13ead6013c0","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-05T09:41:19.197Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"277cce42-6195-41e9-8bc1-b13ead6013c0","name":"long-running-qkmps","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:41:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"long-running-qkmps-head-r2zkr","serviceName":"long-running-qkmps-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:41:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:41:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"long-running-qkmps-head-r2zkr","serviceName":"long-running-qkmps-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:41:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:41:19.230Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"1c0e8e26-a934-499e-9cc4-d59416692a0a","rayCluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:19.230Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"346bcd07-1747-4231-9006-5b521a6c009b","cluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:19.231Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"1c0e8e26-a934-499e-9cc4-d59416692a0a","pod":"long-running-qkmps-small-group-worker-pvt7t","ip":"10.132.0.18"}
{"level":"info","ts":"2026-06-05T09:41:19.231Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"1c0e8e26-a934-499e-9cc4-d59416692a0a","pod":"long-running-qkmps-head-r2zkr","ip":"10.133.0.38"}
{"level":"info","ts":"2026-06-05T09:41:19.231Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"bc2dbe83-77fe-4598-9f52-47915a802cd7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:19.231Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"bc2dbe83-77fe-4598-9f52-47915a802cd7","name":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:19.231Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"bc2dbe83-77fe-4598-9f52-47915a802cd7","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:19.231Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"1c0e8e26-a934-499e-9cc4-d59416692a0a","rayCluster":"long-running-qkmps","podIPs":["10.132.0.18","10.133.0.38"]}
{"level":"info","ts":"2026-06-05T09:41:19.231Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"bc2dbe83-77fe-4598-9f52-47915a802cd7"}
{"level":"info","ts":"2026-06-05T09:41:19.231Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"bc2dbe83-77fe-4598-9f52-47915a802cd7","RayCluster":"long-running-qkmps","State":""}
{"level":"info","ts":"2026-06-05T09:41:19.245Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"346bcd07-1747-4231-9006-5b521a6c009b","name":"long-running-qkmps-head"}
{"level":"info","ts":"2026-06-05T09:41:19.258Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"346bcd07-1747-4231-9006-5b521a6c009b","name":"long-running-qkmps-workers"}
{"level":"info","ts":"2026-06-05T09:41:19.258Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"346bcd07-1747-4231-9006-5b521a6c009b","cluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:19.429Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"counter","namespace":"test-ns-l4cdh"},"reconcileID":"2f13e462-dcda-4f95-8e81-5c768b440aa1"}
{"level":"info","ts":"2026-06-05T09:41:19.460Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail","namespace":"test-ns-l4cdh"},"reconcileID":"af38c491-d599-4ef5-aafd-e30cd3aa89cd"}
{"level":"info","ts":"2026-06-05T09:41:19.737Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"17a2cb5c-a958-48fa-8597-e72964af6a0b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:19.737Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"17a2cb5c-a958-48fa-8597-e72964af6a0b","name":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:19.737Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"17a2cb5c-a958-48fa-8597-e72964af6a0b","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:19.737Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"17a2cb5c-a958-48fa-8597-e72964af6a0b"}
{"level":"info","ts":"2026-06-05T09:41:19.737Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"17a2cb5c-a958-48fa-8597-e72964af6a0b","RayCluster":"long-running-qkmps","State":""}
{"level":"info","ts":"2026-06-05T09:41:21.230Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"0090fe3b-26c6-452a-b1a2-fce8a422ccba"}
{"level":"info","ts":"2026-06-05T09:41:21.231Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"0090fe3b-26c6-452a-b1a2-fce8a422ccba"}
{"level":"info","ts":"2026-06-05T09:41:21.231Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"0090fe3b-26c6-452a-b1a2-fce8a422ccba","1 head service found":"long-running-qkmps-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:21.231Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"0090fe3b-26c6-452a-b1a2-fce8a422ccba","Found 1 head Pod":"long-running-qkmps-head-r2zkr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:41:21.231Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"0090fe3b-26c6-452a-b1a2-fce8a422ccba","head Pod":"long-running-qkmps-head-r2zkr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qkmps-head-r2zkr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:21.231Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"0090fe3b-26c6-452a-b1a2-fce8a422ccba","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:41:21.231Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"0090fe3b-26c6-452a-b1a2-fce8a422ccba","worker Pod":"long-running-qkmps-small-group-worker-pvt7t","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qkmps-small-group-worker-pvt7t. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:21.231Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"0090fe3b-26c6-452a-b1a2-fce8a422ccba","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:21.231Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"0090fe3b-26c6-452a-b1a2-fce8a422ccba","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:41:21.231Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"0090fe3b-26c6-452a-b1a2-fce8a422ccba","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:21.231Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"0090fe3b-26c6-452a-b1a2-fce8a422ccba","podName":"long-running-qkmps-head-r2zkr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:41:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:41:21.232Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"0090fe3b-26c6-452a-b1a2-fce8a422ccba","podName":"long-running-qkmps-head-r2zkr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:41:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:41:21.232Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"0090fe3b-26c6-452a-b1a2-fce8a422ccba","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:41:21.232Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"0090fe3b-26c6-452a-b1a2-fce8a422ccba","seconds":300}
{"level":"info","ts":"2026-06-05T09:41:22.738Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"52135192-cbbc-447a-82d5-2b697f8cc147","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:22.738Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"52135192-cbbc-447a-82d5-2b697f8cc147","name":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:22.738Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"52135192-cbbc-447a-82d5-2b697f8cc147","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:22.738Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"52135192-cbbc-447a-82d5-2b697f8cc147"}
{"level":"info","ts":"2026-06-05T09:41:22.738Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"52135192-cbbc-447a-82d5-2b697f8cc147","RayCluster":"long-running-qkmps","State":""}
{"level":"info","ts":"2026-06-05T09:41:25.737Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"37c9f097-372a-4695-8bc6-8c4a6e7a3249"}
{"level":"info","ts":"2026-06-05T09:41:25.737Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"37c9f097-372a-4695-8bc6-8c4a6e7a3249"}
{"level":"info","ts":"2026-06-05T09:41:25.737Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"37c9f097-372a-4695-8bc6-8c4a6e7a3249","1 head service found":"long-running-qkmps-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:25.737Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"37c9f097-372a-4695-8bc6-8c4a6e7a3249","Found 1 head Pod":"long-running-qkmps-head-r2zkr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:41:25.737Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"37c9f097-372a-4695-8bc6-8c4a6e7a3249","head Pod":"long-running-qkmps-head-r2zkr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qkmps-head-r2zkr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:25.737Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"37c9f097-372a-4695-8bc6-8c4a6e7a3249","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:41:25.737Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"37c9f097-372a-4695-8bc6-8c4a6e7a3249","worker Pod":"long-running-qkmps-small-group-worker-pvt7t","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qkmps-small-group-worker-pvt7t. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:25.737Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"37c9f097-372a-4695-8bc6-8c4a6e7a3249","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:25.737Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"37c9f097-372a-4695-8bc6-8c4a6e7a3249","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:41:25.737Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"37c9f097-372a-4695-8bc6-8c4a6e7a3249","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:25.737Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"37c9f097-372a-4695-8bc6-8c4a6e7a3249","podName":"long-running-qkmps-small-group-worker-pvt7t","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:41:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-05T09:41:25.737Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"37c9f097-372a-4695-8bc6-8c4a6e7a3249","podName":"long-running-qkmps-small-group-worker-pvt7t","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:41:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-05T09:41:25.737Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"37c9f097-372a-4695-8bc6-8c4a6e7a3249","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:41:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:41:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:41:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-05T09:41:25.737Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"37c9f097-372a-4695-8bc6-8c4a6e7a3249","name":"long-running-qkmps","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:41:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"long-running-qkmps-head-r2zkr","serviceName":"long-running-qkmps-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:41:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:41:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"long-running-qkmps-head-r2zkr","serviceName":"long-running-qkmps-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:41:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:41:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:41:25.738Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"e9c8d653-b984-4816-b0aa-e1e753bea072","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:25.738Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"e9c8d653-b984-4816-b0aa-e1e753bea072","name":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:25.738Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"e9c8d653-b984-4816-b0aa-e1e753bea072","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:25.738Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"e9c8d653-b984-4816-b0aa-e1e753bea072"}
{"level":"info","ts":"2026-06-05T09:41:25.738Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"e9c8d653-b984-4816-b0aa-e1e753bea072","RayCluster":"long-running-qkmps","State":""}
{"level":"info","ts":"2026-06-05T09:41:25.750Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"8335fe2f-f671-47b7-a892-287e90b56330","cluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:25.750Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"820da416-5fdb-4697-8f3d-37921effe64a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:25.750Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"820da416-5fdb-4697-8f3d-37921effe64a","name":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:25.750Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"c6419e44-7b83-4411-a5d1-68862403cc11","rayCluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:25.750Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"820da416-5fdb-4697-8f3d-37921effe64a","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:25.750Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"c6419e44-7b83-4411-a5d1-68862403cc11","pod":"long-running-qkmps-head-r2zkr","ip":"10.133.0.38"}
{"level":"info","ts":"2026-06-05T09:41:25.750Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"c6419e44-7b83-4411-a5d1-68862403cc11","pod":"long-running-qkmps-small-group-worker-pvt7t","ip":"10.132.0.18"}
{"level":"info","ts":"2026-06-05T09:41:25.750Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"c6419e44-7b83-4411-a5d1-68862403cc11","rayCluster":"long-running-qkmps","podIPs":["10.133.0.38","10.132.0.18"]}
{"level":"info","ts":"2026-06-05T09:41:25.750Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"820da416-5fdb-4697-8f3d-37921effe64a"}
{"level":"info","ts":"2026-06-05T09:41:25.750Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"820da416-5fdb-4697-8f3d-37921effe64a","RayCluster":"long-running-qkmps","State":""}
{"level":"info","ts":"2026-06-05T09:41:25.760Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"8335fe2f-f671-47b7-a892-287e90b56330","name":"long-running-qkmps-head"}
{"level":"info","ts":"2026-06-05T09:41:25.772Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"8335fe2f-f671-47b7-a892-287e90b56330","name":"long-running-qkmps-workers"}
{"level":"info","ts":"2026-06-05T09:41:25.772Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"8335fe2f-f671-47b7-a892-287e90b56330","cluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:27.751Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"2a439a88-7ec9-4a88-8cf9-c19762d114ec"}
{"level":"info","ts":"2026-06-05T09:41:27.751Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"2a439a88-7ec9-4a88-8cf9-c19762d114ec"}
{"level":"info","ts":"2026-06-05T09:41:27.751Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"2a439a88-7ec9-4a88-8cf9-c19762d114ec","1 head service found":"long-running-qkmps-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:27.751Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"2a439a88-7ec9-4a88-8cf9-c19762d114ec","Found 1 head Pod":"long-running-qkmps-head-r2zkr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:41:27.751Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"2a439a88-7ec9-4a88-8cf9-c19762d114ec","head Pod":"long-running-qkmps-head-r2zkr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qkmps-head-r2zkr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:27.751Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"2a439a88-7ec9-4a88-8cf9-c19762d114ec","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:41:27.751Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"2a439a88-7ec9-4a88-8cf9-c19762d114ec","worker Pod":"long-running-qkmps-small-group-worker-pvt7t","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qkmps-small-group-worker-pvt7t. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:27.751Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"2a439a88-7ec9-4a88-8cf9-c19762d114ec","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:27.751Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"2a439a88-7ec9-4a88-8cf9-c19762d114ec","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:41:27.751Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"2a439a88-7ec9-4a88-8cf9-c19762d114ec","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:27.751Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"2a439a88-7ec9-4a88-8cf9-c19762d114ec","podName":"long-running-qkmps-small-group-worker-pvt7t","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:41:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-05T09:41:27.751Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"2a439a88-7ec9-4a88-8cf9-c19762d114ec","podName":"long-running-qkmps-small-group-worker-pvt7t","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:41:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-05T09:41:27.751Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"2a439a88-7ec9-4a88-8cf9-c19762d114ec","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:41:27.751Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"2a439a88-7ec9-4a88-8cf9-c19762d114ec","seconds":300}
{"level":"info","ts":"2026-06-05T09:41:28.739Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"27a047d9-cdcc-4874-91f1-b9f0d8a5f70f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:28.740Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"27a047d9-cdcc-4874-91f1-b9f0d8a5f70f","name":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:28.740Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"27a047d9-cdcc-4874-91f1-b9f0d8a5f70f","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:28.740Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"27a047d9-cdcc-4874-91f1-b9f0d8a5f70f"}
{"level":"info","ts":"2026-06-05T09:41:28.740Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"27a047d9-cdcc-4874-91f1-b9f0d8a5f70f","RayCluster":"long-running-qkmps","State":""}
{"level":"info","ts":"2026-06-05T09:41:30.218Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"c64d8fc4-1cc6-41e5-8317-c8a42cc075cc"}
{"level":"info","ts":"2026-06-05T09:41:30.218Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"c64d8fc4-1cc6-41e5-8317-c8a42cc075cc"}
{"level":"info","ts":"2026-06-05T09:41:30.218Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"c64d8fc4-1cc6-41e5-8317-c8a42cc075cc","1 head service found":"long-running-qkmps-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:30.218Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"c64d8fc4-1cc6-41e5-8317-c8a42cc075cc","Found 1 head Pod":"long-running-qkmps-head-r2zkr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:41:30.218Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"c64d8fc4-1cc6-41e5-8317-c8a42cc075cc","head Pod":"long-running-qkmps-head-r2zkr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qkmps-head-r2zkr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:30.218Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"c64d8fc4-1cc6-41e5-8317-c8a42cc075cc","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:41:30.218Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"c64d8fc4-1cc6-41e5-8317-c8a42cc075cc","worker Pod":"long-running-qkmps-small-group-worker-pvt7t","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qkmps-small-group-worker-pvt7t. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:30.218Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"c64d8fc4-1cc6-41e5-8317-c8a42cc075cc","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:30.218Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"c64d8fc4-1cc6-41e5-8317-c8a42cc075cc","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:41:30.218Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"c64d8fc4-1cc6-41e5-8317-c8a42cc075cc","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:30.218Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"c64d8fc4-1cc6-41e5-8317-c8a42cc075cc","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-06-05T09:41:30.218Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"c64d8fc4-1cc6-41e5-8317-c8a42cc075cc","name":"long-running-qkmps","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:41:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"long-running-qkmps-head-r2zkr","serviceName":"long-running-qkmps-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:41:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:41:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:41:30Z","stateTransitionTimes":{"ready":"2026-06-05T09:41:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.38","serviceIP":"10.133.0.38","podName":"long-running-qkmps-head-r2zkr","serviceName":"long-running-qkmps-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:41:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:41:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:04Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:41:04Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:41:30.231Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"2f54dab4-c73e-44c2-bbfb-6902ce20221d","rayCluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:30.231Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"4a4cff12-fd3e-4e86-8412-74def4fa2c9e","cluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:30.231Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"34bfa754-f0c6-4e86-ac67-5a1c6fe252ca","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:30.231Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"34bfa754-f0c6-4e86-ac67-5a1c6fe252ca","name":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:30.231Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"34bfa754-f0c6-4e86-ac67-5a1c6fe252ca","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:30.231Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"2f54dab4-c73e-44c2-bbfb-6902ce20221d","pod":"long-running-qkmps-head-r2zkr","ip":"10.133.0.38"}
{"level":"info","ts":"2026-06-05T09:41:30.232Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"2f54dab4-c73e-44c2-bbfb-6902ce20221d","pod":"long-running-qkmps-small-group-worker-pvt7t","ip":"10.132.0.18"}
{"level":"info","ts":"2026-06-05T09:41:30.232Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"2f54dab4-c73e-44c2-bbfb-6902ce20221d","rayCluster":"long-running-qkmps","podIPs":["10.133.0.38","10.132.0.18"]}
{"level":"info","ts":"2026-06-05T09:41:30.232Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"34bfa754-f0c6-4e86-ac67-5a1c6fe252ca"}
{"level":"info","ts":"2026-06-05T09:41:30.232Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"34bfa754-f0c6-4e86-ac67-5a1c6fe252ca","head service name":"long-running-qkmps-head-svc","namespace":"test-ns-t5s4r"}
{"level":"info","ts":"2026-06-05T09:41:30.232Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"34bfa754-f0c6-4e86-ac67-5a1c6fe252ca","head service URL":"long-running-qkmps-head-svc.test-ns-t5s4r.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-05T09:41:30.232Z","logger":"controllers.RayJob","msg":"user-provided submitter template is used; the first container is assumed to be the submitter","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"34bfa754-f0c6-4e86-ac67-5a1c6fe252ca"}
{"level":"info","ts":"2026-06-05T09:41:30.232Z","logger":"controllers.RayJob","msg":"No command is specified in the user-provided template. Default command is used","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"34bfa754-f0c6-4e86-ac67-5a1c6fe252ca","command":["if","!","ray","job","status","--address","http://long-running-qkmps-head-svc.test-ns-t5s4r.svc.cluster.local:8265","long-running-wtb66",">/dev/null","2>&1",";","then","ray","job","submit","--address","http://long-running-qkmps-head-svc.test-ns-t5s4r.svc.cluster.local:8265","--no-wait","--submission-id","long-running-wtb66","--","python /home/ray/jobs/long_running.py",";","fi",";","ray","job","logs","--address","http://long-running-qkmps-head-svc.test-ns-t5s4r.svc.cluster.local:8265","--follow","long-running-wtb66"]}
{"level":"info","ts":"2026-06-05T09:41:30.240Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"34bfa754-f0c6-4e86-ac67-5a1c6fe252ca","Kubernetes Job":"long-running"}
{"level":"info","ts":"2026-06-05T09:41:30.240Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"34bfa754-f0c6-4e86-ac67-5a1c6fe252ca","SubmissionMode":"K8sJobMode","RayCluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:30.240Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"34bfa754-f0c6-4e86-ac67-5a1c6fe252ca","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-wtb66","rayClusterName":"long-running-qkmps","jobDeploymentStatus":"Initializing","startTime":"2026-06-05T09:41:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-wtb66","rayClusterName":"long-running-qkmps","dashboardURL":"long-running-qkmps-head-svc.test-ns-t5s4r.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-05T09:41:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:41:30.240Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"34bfa754-f0c6-4e86-ac67-5a1c6fe252ca","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-05T09:41:30.245Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"4a4cff12-fd3e-4e86-8412-74def4fa2c9e","name":"long-running-qkmps-head"}
{"level":"info","ts":"2026-06-05T09:41:30.252Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"2845eff5-2e65-4572-9295-14707f855e21","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:30.252Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"2845eff5-2e65-4572-9295-14707f855e21","name":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:30.252Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"2845eff5-2e65-4572-9295-14707f855e21","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:30.252Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"2845eff5-2e65-4572-9295-14707f855e21"}
{"level":"info","ts":"2026-06-05T09:41:30.252Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"2845eff5-2e65-4572-9295-14707f855e21","head service name":"long-running-qkmps-head-svc","namespace":"test-ns-t5s4r"}
{"level":"info","ts":"2026-06-05T09:41:30.252Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"2845eff5-2e65-4572-9295-14707f855e21","head service URL":"long-running-qkmps-head-svc.test-ns-t5s4r.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-05T09:41:30.252Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job for RayJob already exists","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"2845eff5-2e65-4572-9295-14707f855e21","Kubernetes Job":"long-running"}
{"level":"info","ts":"2026-06-05T09:41:30.252Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"2845eff5-2e65-4572-9295-14707f855e21","SubmissionMode":"K8sJobMode","RayCluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:30.252Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"2845eff5-2e65-4572-9295-14707f855e21","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-wtb66","rayClusterName":"long-running-qkmps","jobDeploymentStatus":"Initializing","startTime":"2026-06-05T09:41:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-wtb66","rayClusterName":"long-running-qkmps","dashboardURL":"long-running-qkmps-head-svc.test-ns-t5s4r.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-05T09:41:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:41:30.252Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"2845eff5-2e65-4572-9295-14707f855e21","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-05T09:41:30.260Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"4a4cff12-fd3e-4e86-8412-74def4fa2c9e","name":"long-running-qkmps-workers"}
{"level":"info","ts":"2026-06-05T09:41:30.260Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"4a4cff12-fd3e-4e86-8412-74def4fa2c9e","cluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:30.260Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"2845eff5-2e65-4572-9295-14707f855e21","error":"Operation cannot be fulfilled on rayjobs.ray.io \"long-running\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-06-05T09:41:30.260Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"2845eff5-2e65-4572-9295-14707f855e21"}
{"level":"error","ts":"2026-06-05T09:41:30.260Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"2845eff5-2e65-4572-9295-14707f855e21","error":"Operation cannot be fulfilled on rayjobs.ray.io \"long-running\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:30.260Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"ee67ead8-58e4-4cdf-833d-c7f2ac99c49e","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:30.260Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"ee67ead8-58e4-4cdf-833d-c7f2ac99c49e","name":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:30.260Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"ee67ead8-58e4-4cdf-833d-c7f2ac99c49e","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:30.260Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"ee67ead8-58e4-4cdf-833d-c7f2ac99c49e"}
{"level":"error","ts":"2026-06-05T09:41:30.311Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"ee67ead8-58e4-4cdf-833d-c7f2ac99c49e","JobId":"long-running-wtb66","error":"Job long-running-wtb66 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:30.312Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"ee67ead8-58e4-4cdf-833d-c7f2ac99c49e"}
{"level":"error","ts":"2026-06-05T09:41:30.312Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"ee67ead8-58e4-4cdf-833d-c7f2ac99c49e","error":"Job long-running-wtb66 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:30.312Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"cee2c1bc-77d8-4f03-bc94-f4ac6e717110","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:30.312Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"cee2c1bc-77d8-4f03-bc94-f4ac6e717110","name":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:30.312Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"cee2c1bc-77d8-4f03-bc94-f4ac6e717110","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:30.312Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"cee2c1bc-77d8-4f03-bc94-f4ac6e717110"}
{"level":"error","ts":"2026-06-05T09:41:30.316Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"cee2c1bc-77d8-4f03-bc94-f4ac6e717110","JobId":"long-running-wtb66","error":"Job long-running-wtb66 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:30.316Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"cee2c1bc-77d8-4f03-bc94-f4ac6e717110"}
{"level":"error","ts":"2026-06-05T09:41:30.316Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"cee2c1bc-77d8-4f03-bc94-f4ac6e717110","error":"Job long-running-wtb66 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:30.323Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"7bcdaf7f-99f0-4583-a8db-5d185e940fa5","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:30.323Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"7bcdaf7f-99f0-4583-a8db-5d185e940fa5","name":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:30.323Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"7bcdaf7f-99f0-4583-a8db-5d185e940fa5","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:30.323Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"7bcdaf7f-99f0-4583-a8db-5d185e940fa5"}
{"level":"error","ts":"2026-06-05T09:41:30.327Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"7bcdaf7f-99f0-4583-a8db-5d185e940fa5","JobId":"long-running-wtb66","error":"Job long-running-wtb66 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:30.328Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"7bcdaf7f-99f0-4583-a8db-5d185e940fa5"}
{"level":"error","ts":"2026-06-05T09:41:30.328Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"7bcdaf7f-99f0-4583-a8db-5d185e940fa5","error":"Job long-running-wtb66 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:30.368Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"b8b2dfaf-df81-4213-82df-8a3af1b40f00","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:30.368Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"b8b2dfaf-df81-4213-82df-8a3af1b40f00","name":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:30.368Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"b8b2dfaf-df81-4213-82df-8a3af1b40f00","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:30.368Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"b8b2dfaf-df81-4213-82df-8a3af1b40f00"}
{"level":"error","ts":"2026-06-05T09:41:30.374Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"b8b2dfaf-df81-4213-82df-8a3af1b40f00","JobId":"long-running-wtb66","error":"Job long-running-wtb66 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:30.374Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"b8b2dfaf-df81-4213-82df-8a3af1b40f00"}
{"level":"error","ts":"2026-06-05T09:41:30.374Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"b8b2dfaf-df81-4213-82df-8a3af1b40f00","error":"Job long-running-wtb66 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:30.454Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"91ced113-1031-4bd4-bb20-69590834e145","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:30.454Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"91ced113-1031-4bd4-bb20-69590834e145","name":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:30.454Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"91ced113-1031-4bd4-bb20-69590834e145","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:30.454Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"91ced113-1031-4bd4-bb20-69590834e145"}
{"level":"error","ts":"2026-06-05T09:41:30.459Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"91ced113-1031-4bd4-bb20-69590834e145","JobId":"long-running-wtb66","error":"Job long-running-wtb66 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:30.460Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"91ced113-1031-4bd4-bb20-69590834e145"}
{"level":"error","ts":"2026-06-05T09:41:30.460Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"91ced113-1031-4bd4-bb20-69590834e145","error":"Job long-running-wtb66 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:30.620Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"1e5f8b51-a792-4614-b6ea-c59bb5e274eb","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:30.620Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"1e5f8b51-a792-4614-b6ea-c59bb5e274eb","name":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:30.621Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"1e5f8b51-a792-4614-b6ea-c59bb5e274eb","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:30.621Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"1e5f8b51-a792-4614-b6ea-c59bb5e274eb"}
{"level":"error","ts":"2026-06-05T09:41:30.626Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"1e5f8b51-a792-4614-b6ea-c59bb5e274eb","JobId":"long-running-wtb66","error":"Job long-running-wtb66 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:30.626Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"1e5f8b51-a792-4614-b6ea-c59bb5e274eb"}
{"level":"error","ts":"2026-06-05T09:41:30.626Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"1e5f8b51-a792-4614-b6ea-c59bb5e274eb","error":"Job long-running-wtb66 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:30.797Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"e34f3632-f39a-4b25-b1a7-762d39a511ba","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:30.797Z","logger":"controllers.RayJob","msg":"Try to transition the status to `Suspending`","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"e34f3632-f39a-4b25-b1a7-762d39a511ba","oldStatus":"Running"}
{"level":"info","ts":"2026-06-05T09:41:30.797Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"e34f3632-f39a-4b25-b1a7-762d39a511ba","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-wtb66","rayClusterName":"long-running-qkmps","dashboardURL":"long-running-qkmps-head-svc.test-ns-t5s4r.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-05T09:41:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-wtb66","rayClusterName":"long-running-qkmps","dashboardURL":"long-running-qkmps-head-svc.test-ns-t5s4r.svc.cluster.local:8265","jobDeploymentStatus":"Suspending","startTime":"2026-06-05T09:41:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:41:30.797Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"e34f3632-f39a-4b25-b1a7-762d39a511ba","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Suspending"}
{"level":"info","ts":"2026-06-05T09:41:30.809Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"6b7b3c7a-8f9b-434c-9ae5-5b8d60a09a3a","JobStatus":"","JobDeploymentStatus":"Suspending","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:30.820Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob is deleted","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"6b7b3c7a-8f9b-434c-9ae5-5b8d60a09a3a","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:30.820Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"6b7b3c7a-8f9b-434c-9ae5-5b8d60a09a3a","isClusterDeleted":false}
{"level":"info","ts":"2026-06-05T09:41:30.821Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"e1a1bbd6-3324-4985-b18f-112d80f27f4f"}
{"level":"info","ts":"2026-06-05T09:41:30.821Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"fbfb4c6f-252c-4c8d-af92-411113d3e47e","namespacedName":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:30.821Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"reconcileID":"c0719e2c-941e-4b0e-a458-c6a50e44e6ba"}
{"level":"info","ts":"2026-06-05T09:41:30.821Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"55b7f83b-fd09-47f4-b4b8-c95dcc6282bf","rayCluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:30.821Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"fbfb4c6f-252c-4c8d-af92-411113d3e47e","cluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:30.821Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"fbfb4c6f-252c-4c8d-af92-411113d3e47e","cluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:30.821Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"fbfb4c6f-252c-4c8d-af92-411113d3e47e","namespace":"test-ns-t5s4r","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-05T09:41:30.821Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"55b7f83b-fd09-47f4-b4b8-c95dcc6282bf","pod":"long-running-qkmps-head-r2zkr","ip":"10.133.0.38"}
{"level":"info","ts":"2026-06-05T09:41:30.821Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"55b7f83b-fd09-47f4-b4b8-c95dcc6282bf","pod":"long-running-qkmps-small-group-worker-pvt7t","ip":"10.132.0.18"}
{"level":"info","ts":"2026-06-05T09:41:30.821Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"55b7f83b-fd09-47f4-b4b8-c95dcc6282bf","rayCluster":"long-running-qkmps","podIPs":["10.133.0.38","10.132.0.18"]}
{"level":"info","ts":"2026-06-05T09:41:30.828Z","logger":"controllers.RayJob","msg":"The associated submitter Kubernetes Job for RayJob is deleted","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"6b7b3c7a-8f9b-434c-9ae5-5b8d60a09a3a","Submitter K8s Job":"long-running"}
{"level":"info","ts":"2026-06-05T09:41:30.828Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"fbfb4c6f-252c-4c8d-af92-411113d3e47e","name":"kuberay-gateway-access","namespace":"test-ns-t5s4r"}
{"level":"info","ts":"2026-06-05T09:41:30.828Z","logger":"controllers.RayJob","msg":"deleteSubmitterJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"6b7b3c7a-8f9b-434c-9ae5-5b8d60a09a3a","isJobDeleted":false}
{"level":"info","ts":"2026-06-05T09:41:30.828Z","logger":"controllers.RayJob","msg":"The release of the compute resources has not been completed yet. Wait for the resources to be deleted before the status transitions to avoid a resource leak.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"6b7b3c7a-8f9b-434c-9ae5-5b8d60a09a3a"}
{"level":"info","ts":"2026-06-05T09:41:30.829Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"cd43c8f9-3c34-45de-a127-6e84b30311d6","JobStatus":"","JobDeploymentStatus":"Suspending","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:30.829Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"cd43c8f9-3c34-45de-a127-6e84b30311d6","RayCluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:30.829Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"cd43c8f9-3c34-45de-a127-6e84b30311d6","isClusterDeleted":false}
{"level":"info","ts":"2026-06-05T09:41:30.829Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job has been already deleted","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"cd43c8f9-3c34-45de-a127-6e84b30311d6","Kubernetes Job":""}
{"level":"info","ts":"2026-06-05T09:41:30.829Z","logger":"controllers.RayJob","msg":"deleteSubmitterJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"cd43c8f9-3c34-45de-a127-6e84b30311d6","isJobDeleted":true}
{"level":"info","ts":"2026-06-05T09:41:30.829Z","logger":"controllers.RayJob","msg":"The release of the compute resources has not been completed yet. Wait for the resources to be deleted before the status transitions to avoid a resource leak.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"cd43c8f9-3c34-45de-a127-6e84b30311d6"}
{"level":"info","ts":"2026-06-05T09:41:30.835Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"fbfb4c6f-252c-4c8d-af92-411113d3e47e","name":"test-ns-t5s4r-long-running-qkmps","namespace":"opendatahub"}
{"level":"info","ts":"2026-06-05T09:41:30.841Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"fbfb4c6f-252c-4c8d-af92-411113d3e47e","configMap":"kube-rbac-proxy-config-long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:30.846Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"fbfb4c6f-252c-4c8d-af92-411113d3e47e","serviceAccount":"long-running-qkmps-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:41:30.853Z","logger":"raycluster-resource","msg":"default","name":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:30.853Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-qkmps","namespace":"test-ns-t5s4r"}
{"level":"info","ts":"2026-06-05T09:41:30.865Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"06339819-3342-4121-870a-fcd0d2bf762b"}
{"level":"info","ts":"2026-06-05T09:41:30.865Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"cedcae77-664e-486f-bf7f-093f3e2e1e13","namespace":"test-ns-t5s4r","clusterName":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:30.865Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"8bce9260-a33d-4b1e-ae50-fd3b39681697","JobStatus":"","JobDeploymentStatus":"Suspending","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:30.865Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"8bce9260-a33d-4b1e-ae50-fd3b39681697","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:30.865Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"8bce9260-a33d-4b1e-ae50-fd3b39681697","isClusterDeleted":true}
{"level":"info","ts":"2026-06-05T09:41:30.865Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job has been already deleted","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"8bce9260-a33d-4b1e-ae50-fd3b39681697","Kubernetes Job":""}
{"level":"info","ts":"2026-06-05T09:41:30.865Z","logger":"controllers.RayJob","msg":"deleteSubmitterJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"8bce9260-a33d-4b1e-ae50-fd3b39681697","isJobDeleted":true}
{"level":"info","ts":"2026-06-05T09:41:30.865Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"8bce9260-a33d-4b1e-ae50-fd3b39681697","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-wtb66","rayClusterName":"long-running-qkmps","dashboardURL":"long-running-qkmps-head-svc.test-ns-t5s4r.svc.cluster.local:8265","jobDeploymentStatus":"Suspending","startTime":"2026-06-05T09:41:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobDeploymentStatus":"Suspended","startTime":"2026-06-05T09:41:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:41:30.865Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"8bce9260-a33d-4b1e-ae50-fd3b39681697","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Suspending","new JobDeploymentStatus":"Suspended"}
{"level":"info","ts":"2026-06-05T09:41:30.906Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"fbfb4c6f-252c-4c8d-af92-411113d3e47e","cluster":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:30.906Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"2697b26a-4aab-4371-b1b9-df58e5a51a16","namespacedName":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:30.906Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"2697b26a-4aab-4371-b1b9-df58e5a51a16"}
{"level":"info","ts":"2026-06-05T09:41:30.906Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"2697b26a-4aab-4371-b1b9-df58e5a51a16","namespace":"test-ns-t5s4r","authClusters":0}
{"level":"info","ts":"2026-06-05T09:41:30.906Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"2697b26a-4aab-4371-b1b9-df58e5a51a16","namespace":"test-ns-t5s4r"}
{"level":"info","ts":"2026-06-05T09:41:30.906Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"8c04ceec-5cef-4e39-b2c0-4e5b88aca3b3","namespacedName":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:30.906Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"8c04ceec-5cef-4e39-b2c0-4e5b88aca3b3"}
{"level":"info","ts":"2026-06-05T09:41:30.906Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"8c04ceec-5cef-4e39-b2c0-4e5b88aca3b3","namespace":"test-ns-t5s4r","authClusters":0}
{"level":"info","ts":"2026-06-05T09:41:30.906Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"8c04ceec-5cef-4e39-b2c0-4e5b88aca3b3","namespace":"test-ns-t5s4r"}
{"level":"info","ts":"2026-06-05T09:41:30.906Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"9d5a6d51-d1bd-43db-ac40-85627c23c64c"}
{"level":"info","ts":"2026-06-05T09:41:30.908Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"1e745cfc-28d9-4b45-b990-0d53a15df0e1","JobStatus":"","JobDeploymentStatus":"Suspended","SubmissionMode":"K8sJobMode"}
{"level":"error","ts":"2026-06-05T09:41:30.909Z","msg":"Failed to delete SelfSigned issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"cedcae77-664e-486f-bf7f-093f3e2e1e13","issuer":"ray-selfsigned-issuer-long-running-qkmps","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-qkmps\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:254\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:30.929Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"cedcae77-664e-486f-bf7f-093f3e2e1e13","errorCount":1}
{"level":"info","ts":"2026-06-05T09:41:30.929Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"cedcae77-664e-486f-bf7f-093f3e2e1e13"}
{"level":"error","ts":"2026-06-05T09:41:30.929Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"cedcae77-664e-486f-bf7f-093f3e2e1e13","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-qkmps\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:30.934Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"e346e59d-0087-4530-a0ce-084a5c95461a","namespace":"test-ns-t5s4r","clusterName":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:30.934Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qkmps","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-qkmps","reconcileID":"e346e59d-0087-4530-a0ce-084a5c95461a","namespace":"test-ns-t5s4r","clusterName":"long-running-qkmps"}
{"level":"info","ts":"2026-06-05T09:41:30.946Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"0abe218a-e162-4471-bbf6-fd288357d6ae","JobStatus":"","JobDeploymentStatus":"Suspended","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:31.832Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"236afdfd-37f7-44b1-80fd-7645db903d7d","JobStatus":"","JobDeploymentStatus":"Suspended","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:31.832Z","logger":"controllers.RayJob","msg":"The status is 'Suspended', but the suspend flag is false. Transition the status to 'New'.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"236afdfd-37f7-44b1-80fd-7645db903d7d"}
{"level":"info","ts":"2026-06-05T09:41:31.832Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"236afdfd-37f7-44b1-80fd-7645db903d7d","oldRayJobStatus":{"rayJobInfo":{},"jobDeploymentStatus":"Suspended","startTime":"2026-06-05T09:41:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"startTime":"2026-06-05T09:41:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:41:31.832Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"236afdfd-37f7-44b1-80fd-7645db903d7d","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Suspended","new JobDeploymentStatus":""}
{"level":"info","ts":"2026-06-05T09:41:31.844Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"dfee76d3-f322-422e-b12e-2b47bc3812f0","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:31.844Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"dfee76d3-f322-422e-b12e-2b47bc3812f0"}
{"level":"info","ts":"2026-06-05T09:41:31.844Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"dfee76d3-f322-422e-b12e-2b47bc3812f0","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-05T09:41:31.844Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"dfee76d3-f322-422e-b12e-2b47bc3812f0","oldRayJobStatus":{"rayJobInfo":{},"startTime":"2026-06-05T09:41:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r2shw","rayClusterName":"long-running-5s9vc","jobDeploymentStatus":"Initializing","startTime":"2026-06-05T09:41:31Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:41:31.844Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"dfee76d3-f322-422e-b12e-2b47bc3812f0","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-05T09:41:31.856Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"5876a7fa-b186-4887-821f-aa6ba6a56c4c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:31.856Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"5876a7fa-b186-4887-821f-aa6ba6a56c4c","name":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:31.856Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"5876a7fa-b186-4887-821f-aa6ba6a56c4c","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:31.856Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"5876a7fa-b186-4887-821f-aa6ba6a56c4c","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:31.862Z","logger":"raycluster-resource","msg":"default","name":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:31.862Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}
{"level":"info","ts":"2026-06-05T09:41:31.872Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"3263187d-61bb-4fcb-b426-5a07daa17bf5","namespacedName":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:31.872Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"4b3a2f00-dd7e-4068-9968-ef8b07b29632","cluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:31.872Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"3263187d-61bb-4fcb-b426-5a07daa17bf5","mode":"IntegratedOAuth","cluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:31.872Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"3263187d-61bb-4fcb-b426-5a07daa17bf5","cluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:31.872Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"2bab7309-5b4e-4499-9b12-3c120419c56e","rayCluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:31.873Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"3e69ede0-3c1b-4f2a-9a3f-bfa30aee36e9"}
{"level":"info","ts":"2026-06-05T09:41:31.873Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"3263187d-61bb-4fcb-b426-5a07daa17bf5","cluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:31.873Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"2bab7309-5b4e-4499-9b12-3c120419c56e","rayCluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:31.873Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"5876a7fa-b186-4887-821f-aa6ba6a56c4c","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:31.873Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"3e69ede0-3c1b-4f2a-9a3f-bfa30aee36e9"}
{"level":"info","ts":"2026-06-05T09:41:31.873Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"5876a7fa-b186-4887-821f-aa6ba6a56c4c"}
{"level":"info","ts":"2026-06-05T09:41:31.873Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"5876a7fa-b186-4887-821f-aa6ba6a56c4c","RayCluster":"long-running-5s9vc","State":""}
{"level":"info","ts":"2026-06-05T09:41:31.873Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"65ff0767-4d8a-4423-bfce-a760574243dc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:31.873Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"65ff0767-4d8a-4423-bfce-a760574243dc","name":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:31.873Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"65ff0767-4d8a-4423-bfce-a760574243dc","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:31.873Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"65ff0767-4d8a-4423-bfce-a760574243dc"}
{"level":"info","ts":"2026-06-05T09:41:31.873Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"65ff0767-4d8a-4423-bfce-a760574243dc","RayCluster":"long-running-5s9vc","State":""}
{"level":"info","ts":"2026-06-05T09:41:31.879Z","logger":"raycluster-resource","msg":"default","name":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:31.879Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}
{"level":"info","ts":"2026-06-05T09:41:31.906Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"3e69ede0-3c1b-4f2a-9a3f-bfa30aee36e9","name":"long-running-5s9vc-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:31.906Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"2bab7309-5b4e-4499-9b12-3c120419c56e","rayCluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:31.906Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"2bab7309-5b4e-4499-9b12-3c120419c56e","rayCluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:31.906Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"3e69ede0-3c1b-4f2a-9a3f-bfa30aee36e9","error":"head certificate secret ray-head-secret-long-running-5s9vc not found"}
{"level":"error","ts":"2026-06-05T09:41:31.906Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"3e69ede0-3c1b-4f2a-9a3f-bfa30aee36e9","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5s9vc not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:31.906Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"3e69ede0-3c1b-4f2a-9a3f-bfa30aee36e9","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5s9vc not found"}
{"level":"info","ts":"2026-06-05T09:41:31.906Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"3e69ede0-3c1b-4f2a-9a3f-bfa30aee36e9","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-5s9vc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:31.906Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"3e69ede0-3c1b-4f2a-9a3f-bfa30aee36e9","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-5s9vc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:31.906Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"fbcad518-ad57-41d3-b7b0-8631785468c9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:31.906Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"fbcad518-ad57-41d3-b7b0-8631785468c9","name":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:31.906Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"3e69ede0-3c1b-4f2a-9a3f-bfa30aee36e9","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-5s9vc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:31.906Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"3e69ede0-3c1b-4f2a-9a3f-bfa30aee36e9","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-05T09:41:31.906Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"3e69ede0-3c1b-4f2a-9a3f-bfa30aee36e9","name":"long-running-5s9vc","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:41:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-5s9vc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:41:31.906Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"fbcad518-ad57-41d3-b7b0-8631785468c9","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:31.907Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"fbcad518-ad57-41d3-b7b0-8631785468c9"}
{"level":"info","ts":"2026-06-05T09:41:31.907Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"fbcad518-ad57-41d3-b7b0-8631785468c9","RayCluster":"long-running-5s9vc","State":""}
{"level":"info","ts":"2026-06-05T09:41:31.914Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"4b3a2f00-dd7e-4068-9968-ef8b07b29632","cluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:31.914Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"62ac3f88-4a10-4bb5-9082-1969e01a1574","cluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:31.914Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"3263187d-61bb-4fcb-b426-5a07daa17bf5","name":"long-running-5s9vc-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:41:31.914Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"3e69ede0-3c1b-4f2a-9a3f-bfa30aee36e9","name":"long-running-5s9vc","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-5s9vc\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-t5s4r","name":"long-running-5s9vc"}}
{"level":"info","ts":"2026-06-05T09:41:31.914Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"3e69ede0-3c1b-4f2a-9a3f-bfa30aee36e9"}
{"level":"error","ts":"2026-06-05T09:41:31.915Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"3e69ede0-3c1b-4f2a-9a3f-bfa30aee36e9","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5s9vc not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:31.915Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"715edd72-4fe1-4682-9c52-3852e94477fd"}
{"level":"info","ts":"2026-06-05T09:41:31.915Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"715edd72-4fe1-4682-9c52-3852e94477fd"}
{"level":"info","ts":"2026-06-05T09:41:31.915Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"715edd72-4fe1-4682-9c52-3852e94477fd","1 head service found":"long-running-5s9vc-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:31.915Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"715edd72-4fe1-4682-9c52-3852e94477fd","error":"head certificate secret ray-head-secret-long-running-5s9vc not found"}
{"level":"error","ts":"2026-06-05T09:41:31.915Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"715edd72-4fe1-4682-9c52-3852e94477fd","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5s9vc not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:31.915Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"715edd72-4fe1-4682-9c52-3852e94477fd","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5s9vc not found"}
{"level":"info","ts":"2026-06-05T09:41:31.915Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"715edd72-4fe1-4682-9c52-3852e94477fd","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-5s9vc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:31.915Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"715edd72-4fe1-4682-9c52-3852e94477fd","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-5s9vc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:31.915Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"715edd72-4fe1-4682-9c52-3852e94477fd","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-5s9vc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:31.915Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"715edd72-4fe1-4682-9c52-3852e94477fd","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-05T09:41:31.915Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"715edd72-4fe1-4682-9c52-3852e94477fd","name":"long-running-5s9vc","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:41:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-5s9vc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:41:31.918Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"2bab7309-5b4e-4499-9b12-3c120419c56e"}
{"level":"info","ts":"2026-06-05T09:41:31.918Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"2bab7309-5b4e-4499-9b12-3c120419c56e","rayCluster":"long-running-5s9vc","secret":"ca-secret"}
{"level":"info","ts":"2026-06-05T09:41:31.918Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"2bab7309-5b4e-4499-9b12-3c120419c56e","rayCluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:32.006Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"3263187d-61bb-4fcb-b426-5a07daa17bf5","name":"kuberay-gateway-access","namespace":"test-ns-t5s4r","operation":"created"}
{"level":"info","ts":"2026-06-05T09:41:32.006Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"2bab7309-5b4e-4499-9b12-3c120419c56e","rayCluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:32.006Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"715edd72-4fe1-4682-9c52-3852e94477fd"}
{"level":"error","ts":"2026-06-05T09:41:32.006Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"715edd72-4fe1-4682-9c52-3852e94477fd","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5s9vc not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:32.006Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c132af29-3535-4ffa-872d-2cd991c2732e"}
{"level":"info","ts":"2026-06-05T09:41:32.006Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c132af29-3535-4ffa-872d-2cd991c2732e"}
{"level":"info","ts":"2026-06-05T09:41:32.006Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c132af29-3535-4ffa-872d-2cd991c2732e","1 head service found":"long-running-5s9vc-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:32.006Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c132af29-3535-4ffa-872d-2cd991c2732e","error":"head certificate secret ray-head-secret-long-running-5s9vc not found"}
{"level":"error","ts":"2026-06-05T09:41:32.006Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c132af29-3535-4ffa-872d-2cd991c2732e","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5s9vc not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:32.006Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c132af29-3535-4ffa-872d-2cd991c2732e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5s9vc not found"}
{"level":"info","ts":"2026-06-05T09:41:32.006Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c132af29-3535-4ffa-872d-2cd991c2732e","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-5s9vc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:32.007Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c132af29-3535-4ffa-872d-2cd991c2732e","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-5s9vc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:32.007Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c132af29-3535-4ffa-872d-2cd991c2732e","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-5s9vc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:32.007Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c132af29-3535-4ffa-872d-2cd991c2732e","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-05T09:41:32.007Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c132af29-3535-4ffa-872d-2cd991c2732e","name":"long-running-5s9vc","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:41:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-5s9vc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:32Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:32Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:32Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:32Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:41:32.007Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"6198fc62-4a16-45b0-93e0-c44d70ad0d2b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:32.007Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"6198fc62-4a16-45b0-93e0-c44d70ad0d2b","name":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:32.007Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"6198fc62-4a16-45b0-93e0-c44d70ad0d2b","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:32.007Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"6198fc62-4a16-45b0-93e0-c44d70ad0d2b"}
{"level":"info","ts":"2026-06-05T09:41:32.007Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"6198fc62-4a16-45b0-93e0-c44d70ad0d2b","RayCluster":"long-running-5s9vc","State":""}
{"level":"info","ts":"2026-06-05T09:41:32.011Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"62ac3f88-4a10-4bb5-9082-1969e01a1574","name":"long-running-5s9vc-head"}
{"level":"info","ts":"2026-06-05T09:41:32.014Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c132af29-3535-4ffa-872d-2cd991c2732e","name":"long-running-5s9vc","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-5s9vc\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-t5s4r","name":"long-running-5s9vc"}}
{"level":"info","ts":"2026-06-05T09:41:32.014Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c132af29-3535-4ffa-872d-2cd991c2732e"}
{"level":"error","ts":"2026-06-05T09:41:32.014Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c132af29-3535-4ffa-872d-2cd991c2732e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5s9vc not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:32.015Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"3263187d-61bb-4fcb-b426-5a07daa17bf5","name":"test-ns-t5s4r-long-running-5s9vc","operation":"created"}
{"level":"info","ts":"2026-06-05T09:41:32.016Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"dfb723c9-48ea-4a20-8066-800b9cc6ef1e"}
{"level":"info","ts":"2026-06-05T09:41:32.016Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"dfb723c9-48ea-4a20-8066-800b9cc6ef1e"}
{"level":"info","ts":"2026-06-05T09:41:32.016Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"dfb723c9-48ea-4a20-8066-800b9cc6ef1e","1 head service found":"long-running-5s9vc-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:32.016Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"dfb723c9-48ea-4a20-8066-800b9cc6ef1e","error":"head certificate secret ray-head-secret-long-running-5s9vc not found"}
{"level":"error","ts":"2026-06-05T09:41:32.016Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"dfb723c9-48ea-4a20-8066-800b9cc6ef1e","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5s9vc not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:32.016Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"dfb723c9-48ea-4a20-8066-800b9cc6ef1e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5s9vc not found"}
{"level":"info","ts":"2026-06-05T09:41:32.016Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"dfb723c9-48ea-4a20-8066-800b9cc6ef1e","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-5s9vc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:32.017Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"dfb723c9-48ea-4a20-8066-800b9cc6ef1e","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-5s9vc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:32.017Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"dfb723c9-48ea-4a20-8066-800b9cc6ef1e","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-5s9vc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:32.017Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"dfb723c9-48ea-4a20-8066-800b9cc6ef1e"}
{"level":"error","ts":"2026-06-05T09:41:32.017Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"dfb723c9-48ea-4a20-8066-800b9cc6ef1e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5s9vc not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:32.021Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"2bab7309-5b4e-4499-9b12-3c120419c56e"}
{"level":"info","ts":"2026-06-05T09:41:32.021Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"2bab7309-5b4e-4499-9b12-3c120419c56e","rayCluster":"long-running-5s9vc","podIPs":[]}
{"level":"info","ts":"2026-06-05T09:41:32.105Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ffef4fb4-b90f-4910-b5c1-d1b54c757b3d"}
{"level":"info","ts":"2026-06-05T09:41:32.106Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ffef4fb4-b90f-4910-b5c1-d1b54c757b3d"}
{"level":"info","ts":"2026-06-05T09:41:32.106Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ffef4fb4-b90f-4910-b5c1-d1b54c757b3d","1 head service found":"long-running-5s9vc-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:32.106Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ffef4fb4-b90f-4910-b5c1-d1b54c757b3d","error":"head certificate secret ray-head-secret-long-running-5s9vc not found"}
{"level":"info","ts":"2026-06-05T09:41:32.106Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"3263187d-61bb-4fcb-b426-5a07daa17bf5","name":"kube-rbac-proxy-config-long-running-5s9vc","operation":"created"}
{"level":"error","ts":"2026-06-05T09:41:32.106Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ffef4fb4-b90f-4910-b5c1-d1b54c757b3d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5s9vc not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:32.106Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ffef4fb4-b90f-4910-b5c1-d1b54c757b3d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5s9vc not found"}
{"level":"info","ts":"2026-06-05T09:41:32.106Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"2bab7309-5b4e-4499-9b12-3c120419c56e"}
{"level":"info","ts":"2026-06-05T09:41:32.106Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"2bab7309-5b4e-4499-9b12-3c120419c56e","rayCluster":"long-running-5s9vc","podIPs":[]}
{"level":"error","ts":"2026-06-05T09:41:32.106Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"2bab7309-5b4e-4499-9b12-3c120419c56e","error":"Certificate.cert-manager.io \"ray-worker-cert-long-running-5s9vc\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:32.106Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"2bab7309-5b4e-4499-9b12-3c120419c56e"}
{"level":"info","ts":"2026-06-05T09:41:32.106Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"62ac3f88-4a10-4bb5-9082-1969e01a1574","name":"long-running-5s9vc-workers"}
{"level":"info","ts":"2026-06-05T09:41:32.106Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ffef4fb4-b90f-4910-b5c1-d1b54c757b3d","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-5s9vc","ray.io/node-type":"head"}]}
{"level":"error","ts":"2026-06-05T09:41:32.106Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"2bab7309-5b4e-4499-9b12-3c120419c56e","error":"Certificate.cert-manager.io \"ray-worker-cert-long-running-5s9vc\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:32.106Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ffef4fb4-b90f-4910-b5c1-d1b54c757b3d","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-5s9vc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:32.106Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"6920f473-9578-414f-9677-b9f3edbcf78c","rayCluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:32.106Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ffef4fb4-b90f-4910-b5c1-d1b54c757b3d","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-5s9vc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:32.106Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ffef4fb4-b90f-4910-b5c1-d1b54c757b3d"}
{"level":"info","ts":"2026-06-05T09:41:32.106Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"62ac3f88-4a10-4bb5-9082-1969e01a1574","cluster":"long-running-5s9vc"}
{"level":"error","ts":"2026-06-05T09:41:32.106Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ffef4fb4-b90f-4910-b5c1-d1b54c757b3d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5s9vc not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:32.106Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"6920f473-9578-414f-9677-b9f3edbcf78c"}
{"level":"info","ts":"2026-06-05T09:41:32.106Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"51cda9bc-6585-41f6-9ff5-feed0c9c6175","cluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:32.112Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"bdaca87e-3cdc-4ce0-8a91-740280d1833a","rayCluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:32.112Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"bdaca87e-3cdc-4ce0-8a91-740280d1833a"}
{"level":"error","ts":"2026-06-05T09:41:32.114Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"3263187d-61bb-4fcb-b426-5a07daa17bf5","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-5s9vc\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:32.114Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"3263187d-61bb-4fcb-b426-5a07daa17bf5"}
{"level":"error","ts":"2026-06-05T09:41:32.114Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"3263187d-61bb-4fcb-b426-5a07daa17bf5","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-5s9vc\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:32.114Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"835492e7-709e-4e6b-a0a1-9b4c4da27506","namespacedName":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:32.114Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"835492e7-709e-4e6b-a0a1-9b4c4da27506","mode":"IntegratedOAuth","cluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:32.114Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"835492e7-709e-4e6b-a0a1-9b4c4da27506","cluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:32.118Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"51cda9bc-6585-41f6-9ff5-feed0c9c6175","name":"long-running-5s9vc-head"}
{"level":"info","ts":"2026-06-05T09:41:32.130Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"835492e7-709e-4e6b-a0a1-9b4c4da27506","cluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:32.130Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"2d8543f8-c06e-4514-a334-69bf0ae53b0c","namespacedName":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:32.130Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"2d8543f8-c06e-4514-a334-69bf0ae53b0c","mode":"IntegratedOAuth","cluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:32.130Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"2d8543f8-c06e-4514-a334-69bf0ae53b0c","cluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:32.132Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"51cda9bc-6585-41f6-9ff5-feed0c9c6175","name":"long-running-5s9vc-workers"}
{"level":"info","ts":"2026-06-05T09:41:32.133Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"51cda9bc-6585-41f6-9ff5-feed0c9c6175","cluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:32.206Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"af63ad8f-53aa-43ae-980e-75ecac743178"}
{"level":"info","ts":"2026-06-05T09:41:32.206Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"af63ad8f-53aa-43ae-980e-75ecac743178"}
{"level":"info","ts":"2026-06-05T09:41:32.206Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"af63ad8f-53aa-43ae-980e-75ecac743178","1 head service found":"long-running-5s9vc-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:32.206Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"dc45737b-7a1a-4ec4-a57c-2ba14a5ec44f","cluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:32.206Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"af63ad8f-53aa-43ae-980e-75ecac743178","error":"head certificate secret ray-head-secret-long-running-5s9vc not found"}
{"level":"info","ts":"2026-06-05T09:41:32.206Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"a630eee4-2328-4b1f-9c4b-8d6688d938e1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:32.206Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"a630eee4-2328-4b1f-9c4b-8d6688d938e1","name":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:32.206Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"a630eee4-2328-4b1f-9c4b-8d6688d938e1","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"error","ts":"2026-06-05T09:41:32.206Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"af63ad8f-53aa-43ae-980e-75ecac743178","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5s9vc not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:32.206Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"af63ad8f-53aa-43ae-980e-75ecac743178","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5s9vc not found"}
{"level":"info","ts":"2026-06-05T09:41:32.206Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"af63ad8f-53aa-43ae-980e-75ecac743178","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-5s9vc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:32.206Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"2e79abc6-3867-4a4f-8c45-d6eb8d49d019","rayCluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:32.206Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"a630eee4-2328-4b1f-9c4b-8d6688d938e1"}
{"level":"info","ts":"2026-06-05T09:41:32.206Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"a630eee4-2328-4b1f-9c4b-8d6688d938e1","RayCluster":"long-running-5s9vc","State":""}
{"level":"info","ts":"2026-06-05T09:41:32.207Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"af63ad8f-53aa-43ae-980e-75ecac743178","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-5s9vc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:32.206Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"2e79abc6-3867-4a4f-8c45-d6eb8d49d019"}
{"level":"info","ts":"2026-06-05T09:41:32.207Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"af63ad8f-53aa-43ae-980e-75ecac743178","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-5s9vc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:32.207Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"af63ad8f-53aa-43ae-980e-75ecac743178"}
{"level":"error","ts":"2026-06-05T09:41:32.207Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"af63ad8f-53aa-43ae-980e-75ecac743178","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5s9vc not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-05T09:41:32.215Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"2d8543f8-c06e-4514-a334-69bf0ae53b0c","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-5s9vc\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:32.215Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"2d8543f8-c06e-4514-a334-69bf0ae53b0c"}
{"level":"error","ts":"2026-06-05T09:41:32.215Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"2d8543f8-c06e-4514-a334-69bf0ae53b0c","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-5s9vc\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:32.220Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"a99d50e5-ab33-4122-92df-019b089d73b1","namespacedName":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:32.220Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"a99d50e5-ab33-4122-92df-019b089d73b1","mode":"IntegratedOAuth","cluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:32.220Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"a99d50e5-ab33-4122-92df-019b089d73b1","cluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:32.221Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"dc45737b-7a1a-4ec4-a57c-2ba14a5ec44f","name":"long-running-5s9vc-head"}
{"level":"info","ts":"2026-06-05T09:41:32.230Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"a99d50e5-ab33-4122-92df-019b089d73b1","cluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:32.231Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"dc45737b-7a1a-4ec4-a57c-2ba14a5ec44f","name":"long-running-5s9vc-workers"}
{"level":"info","ts":"2026-06-05T09:41:32.231Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"dc45737b-7a1a-4ec4-a57c-2ba14a5ec44f","cluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:32.367Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"28d74dbf-9ca7-4d2e-b1c1-543a87a8f413"}
{"level":"info","ts":"2026-06-05T09:41:32.367Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"28d74dbf-9ca7-4d2e-b1c1-543a87a8f413"}
{"level":"info","ts":"2026-06-05T09:41:32.367Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"28d74dbf-9ca7-4d2e-b1c1-543a87a8f413","1 head service found":"long-running-5s9vc-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:32.367Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"28d74dbf-9ca7-4d2e-b1c1-543a87a8f413","error":"head certificate secret ray-head-secret-long-running-5s9vc not found"}
{"level":"error","ts":"2026-06-05T09:41:32.367Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"28d74dbf-9ca7-4d2e-b1c1-543a87a8f413","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5s9vc not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:32.367Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"28d74dbf-9ca7-4d2e-b1c1-543a87a8f413","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5s9vc not found"}
{"level":"info","ts":"2026-06-05T09:41:32.367Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"28d74dbf-9ca7-4d2e-b1c1-543a87a8f413","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-5s9vc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:32.367Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"28d74dbf-9ca7-4d2e-b1c1-543a87a8f413","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-5s9vc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:32.367Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"28d74dbf-9ca7-4d2e-b1c1-543a87a8f413","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-5s9vc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:32.367Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"28d74dbf-9ca7-4d2e-b1c1-543a87a8f413"}
{"level":"error","ts":"2026-06-05T09:41:32.367Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"28d74dbf-9ca7-4d2e-b1c1-543a87a8f413","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5s9vc not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:32.688Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"20753eb8-bf99-49b8-bba9-6df89b6a6c6a"}
{"level":"info","ts":"2026-06-05T09:41:32.688Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"20753eb8-bf99-49b8-bba9-6df89b6a6c6a"}
{"level":"info","ts":"2026-06-05T09:41:32.688Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"20753eb8-bf99-49b8-bba9-6df89b6a6c6a","1 head service found":"long-running-5s9vc-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:32.689Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"20753eb8-bf99-49b8-bba9-6df89b6a6c6a","error":"head certificate secret ray-head-secret-long-running-5s9vc not found"}
{"level":"error","ts":"2026-06-05T09:41:32.689Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"20753eb8-bf99-49b8-bba9-6df89b6a6c6a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5s9vc not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:32.689Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"20753eb8-bf99-49b8-bba9-6df89b6a6c6a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5s9vc not found"}
{"level":"info","ts":"2026-06-05T09:41:32.689Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"20753eb8-bf99-49b8-bba9-6df89b6a6c6a","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-5s9vc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:32.689Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"20753eb8-bf99-49b8-bba9-6df89b6a6c6a","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-5s9vc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:32.689Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"20753eb8-bf99-49b8-bba9-6df89b6a6c6a","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-5s9vc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:32.689Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"20753eb8-bf99-49b8-bba9-6df89b6a6c6a"}
{"level":"error","ts":"2026-06-05T09:41:32.689Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"20753eb8-bf99-49b8-bba9-6df89b6a6c6a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5s9vc not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:33.329Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"8f1de848-a6f8-43db-8ef4-466eefc76def"}
{"level":"info","ts":"2026-06-05T09:41:33.329Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"8f1de848-a6f8-43db-8ef4-466eefc76def"}
{"level":"info","ts":"2026-06-05T09:41:33.329Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"8f1de848-a6f8-43db-8ef4-466eefc76def","1 head service found":"long-running-5s9vc-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:33.329Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"8f1de848-a6f8-43db-8ef4-466eefc76def","error":"head certificate secret ray-head-secret-long-running-5s9vc not found"}
{"level":"error","ts":"2026-06-05T09:41:33.329Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"8f1de848-a6f8-43db-8ef4-466eefc76def","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5s9vc not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:33.329Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"8f1de848-a6f8-43db-8ef4-466eefc76def","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5s9vc not found"}
{"level":"info","ts":"2026-06-05T09:41:33.329Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"8f1de848-a6f8-43db-8ef4-466eefc76def","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-5s9vc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:33.330Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"8f1de848-a6f8-43db-8ef4-466eefc76def","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-5s9vc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:33.330Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"8f1de848-a6f8-43db-8ef4-466eefc76def","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"long-running-5s9vc","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:41:33.330Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"8f1de848-a6f8-43db-8ef4-466eefc76def"}
{"level":"error","ts":"2026-06-05T09:41:33.330Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"8f1de848-a6f8-43db-8ef4-466eefc76def","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-5s9vc not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:33.948Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"eda0d9ea-f987-4aee-b272-b558884920f7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:33.948Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"eda0d9ea-f987-4aee-b272-b558884920f7","name":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:33.948Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"eda0d9ea-f987-4aee-b272-b558884920f7","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:33.948Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"eda0d9ea-f987-4aee-b272-b558884920f7"}
{"level":"info","ts":"2026-06-05T09:41:33.948Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"eda0d9ea-f987-4aee-b272-b558884920f7","RayCluster":"long-running-5s9vc","State":""}
{"level":"info","ts":"2026-06-05T09:41:34.611Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a5177773-b9b2-47f6-af9f-c57757286efe"}
{"level":"info","ts":"2026-06-05T09:41:34.611Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a5177773-b9b2-47f6-af9f-c57757286efe"}
{"level":"info","ts":"2026-06-05T09:41:34.611Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a5177773-b9b2-47f6-af9f-c57757286efe","1 head service found":"long-running-5s9vc-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:34.611Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a5177773-b9b2-47f6-af9f-c57757286efe"}
{"level":"info","ts":"2026-06-05T09:41:34.611Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a5177773-b9b2-47f6-af9f-c57757286efe","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:41:34.611Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a5177773-b9b2-47f6-af9f-c57757286efe"}
{"level":"info","ts":"2026-06-05T09:41:34.611Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a5177773-b9b2-47f6-af9f-c57757286efe","mode":"IntegratedOAuth","cluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:34.611Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a5177773-b9b2-47f6-af9f-c57757286efe","cluster":"long-running-5s9vc","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:41:34.611Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a5177773-b9b2-47f6-af9f-c57757286efe","cluster":"long-running-5s9vc","authType":"IntegratedOAuth","serviceAccount":"long-running-5s9vc-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-05T09:41:34.611Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a5177773-b9b2-47f6-af9f-c57757286efe","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"long-running-5s9vc","ray.io/group":"headgroup","ray.io/identifier":"long-running-5s9vc-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-05T09:41:34.611Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a5177773-b9b2-47f6-af9f-c57757286efe","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"long-running-5s9vc-head-svc.test-ns-t5s4r.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-05T09:41:34.611Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a5177773-b9b2-47f6-af9f-c57757286efe","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-5s9vc-head-svc.test-ns-t5s4r.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-05T09:41:34.611Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a5177773-b9b2-47f6-af9f-c57757286efe","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-5s9vc-head-svc.test-ns-t5s4r.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-05T09:41:34.611Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a5177773-b9b2-47f6-af9f-c57757286efe","enabled":true}
{"level":"info","ts":"2026-06-05T09:41:34.628Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a5177773-b9b2-47f6-af9f-c57757286efe","name":"long-running-5s9vc-head-r7bq6"}
{"level":"info","ts":"2026-06-05T09:41:34.628Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a5177773-b9b2-47f6-af9f-c57757286efe","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:41:34.628Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a5177773-b9b2-47f6-af9f-c57757286efe","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:34.628Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a5177773-b9b2-47f6-af9f-c57757286efe","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-05T09:41:34.628Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a5177773-b9b2-47f6-af9f-c57757286efe","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:34.628Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a5177773-b9b2-47f6-af9f-c57757286efe","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-05T09:41:34.629Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a5177773-b9b2-47f6-af9f-c57757286efe"}
{"level":"info","ts":"2026-06-05T09:41:34.629Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a5177773-b9b2-47f6-af9f-c57757286efe","nodeType":"worker","rayStartParams":{"address":"long-running-5s9vc-head-svc.test-ns-t5s4r.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-05T09:41:34.629Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a5177773-b9b2-47f6-af9f-c57757286efe","rayStartCmd":"ray start  --address=long-running-5s9vc-head-svc.test-ns-t5s4r.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:41:34.629Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a5177773-b9b2-47f6-af9f-c57757286efe","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-5s9vc-head-svc.test-ns-t5s4r.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:41:34.629Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a5177773-b9b2-47f6-af9f-c57757286efe","enabled":true}
{"level":"info","ts":"2026-06-05T09:41:34.642Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a5177773-b9b2-47f6-af9f-c57757286efe","name":"long-running-5s9vc-small-group-worker-pldls"}
{"level":"info","ts":"2026-06-05T09:41:34.642Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a5177773-b9b2-47f6-af9f-c57757286efe","podName":"long-running-5s9vc-head-r7bq6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:41:34.642Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a5177773-b9b2-47f6-af9f-c57757286efe","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"long-running-5s9vc-head-svc"},"newHead":{"podName":"long-running-5s9vc-head-r7bq6","serviceName":"long-running-5s9vc-head-svc"}}
{"level":"info","ts":"2026-06-05T09:41:34.642Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a5177773-b9b2-47f6-af9f-c57757286efe","name":"long-running-5s9vc","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:41:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-5s9vc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:41:32Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:41:34Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-5s9vc-head-r7bq6","serviceName":"long-running-5s9vc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:41:32Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:41:34.655Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"686f90be-56c4-4155-9115-ad3edc39a8d0","cluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:34.655Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"7f6aca27-412a-49a1-b411-8b4b20ded75f","rayCluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:34.655Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"7f6aca27-412a-49a1-b411-8b4b20ded75f","rayCluster":"long-running-5s9vc","podIPs":[]}
{"level":"info","ts":"2026-06-05T09:41:34.656Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"f0d8a914-2a51-408d-ae7b-0ea74b62e699","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:34.656Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"f0d8a914-2a51-408d-ae7b-0ea74b62e699","name":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:34.656Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"f0d8a914-2a51-408d-ae7b-0ea74b62e699","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:34.656Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"f0d8a914-2a51-408d-ae7b-0ea74b62e699"}
{"level":"info","ts":"2026-06-05T09:41:34.656Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"f0d8a914-2a51-408d-ae7b-0ea74b62e699","RayCluster":"long-running-5s9vc","State":""}
{"level":"info","ts":"2026-06-05T09:41:34.656Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"6d21c98e-dba4-47d9-97fe-61b6ae30dded"}
{"level":"info","ts":"2026-06-05T09:41:34.656Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"6d21c98e-dba4-47d9-97fe-61b6ae30dded"}
{"level":"info","ts":"2026-06-05T09:41:34.656Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"6d21c98e-dba4-47d9-97fe-61b6ae30dded","1 head service found":"long-running-5s9vc-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:34.656Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"6d21c98e-dba4-47d9-97fe-61b6ae30dded","Found 1 head Pod":"long-running-5s9vc-head-r7bq6","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:41:34.656Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"6d21c98e-dba4-47d9-97fe-61b6ae30dded","head Pod":"long-running-5s9vc-head-r7bq6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5s9vc-head-r7bq6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:34.656Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"6d21c98e-dba4-47d9-97fe-61b6ae30dded","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:41:34.656Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"6d21c98e-dba4-47d9-97fe-61b6ae30dded","worker Pod":"long-running-5s9vc-small-group-worker-pldls","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-5s9vc-small-group-worker-pldls. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:34.656Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"6d21c98e-dba4-47d9-97fe-61b6ae30dded","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:34.656Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"6d21c98e-dba4-47d9-97fe-61b6ae30dded","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:41:34.656Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"6d21c98e-dba4-47d9-97fe-61b6ae30dded","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:34.656Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"6d21c98e-dba4-47d9-97fe-61b6ae30dded","podName":"long-running-5s9vc-head-r7bq6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:41:34.656Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"6d21c98e-dba4-47d9-97fe-61b6ae30dded","podName":"long-running-5s9vc-head-r7bq6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:41:34.656Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"6d21c98e-dba4-47d9-97fe-61b6ae30dded","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:41:34.656Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"6d21c98e-dba4-47d9-97fe-61b6ae30dded","seconds":300}
{"level":"info","ts":"2026-06-05T09:41:34.659Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ffd19109-a98e-442f-a207-3034bc98aa25"}
{"level":"info","ts":"2026-06-05T09:41:34.659Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ffd19109-a98e-442f-a207-3034bc98aa25"}
{"level":"info","ts":"2026-06-05T09:41:34.659Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ffd19109-a98e-442f-a207-3034bc98aa25","1 head service found":"long-running-5s9vc-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:34.660Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ffd19109-a98e-442f-a207-3034bc98aa25","Found 1 head Pod":"long-running-5s9vc-head-r7bq6","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:41:34.660Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ffd19109-a98e-442f-a207-3034bc98aa25","head Pod":"long-running-5s9vc-head-r7bq6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5s9vc-head-r7bq6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:34.660Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ffd19109-a98e-442f-a207-3034bc98aa25","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:41:34.660Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ffd19109-a98e-442f-a207-3034bc98aa25","worker Pod":"long-running-5s9vc-small-group-worker-pldls","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-5s9vc-small-group-worker-pldls. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:34.660Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ffd19109-a98e-442f-a207-3034bc98aa25","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:34.660Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ffd19109-a98e-442f-a207-3034bc98aa25","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:41:34.660Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ffd19109-a98e-442f-a207-3034bc98aa25","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:34.660Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ffd19109-a98e-442f-a207-3034bc98aa25","podName":"long-running-5s9vc-head-r7bq6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:41:34.660Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ffd19109-a98e-442f-a207-3034bc98aa25","podName":"long-running-5s9vc-head-r7bq6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:41:34.660Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ffd19109-a98e-442f-a207-3034bc98aa25","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:41:34.660Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ffd19109-a98e-442f-a207-3034bc98aa25","seconds":300}
{"level":"info","ts":"2026-06-05T09:41:34.667Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"686f90be-56c4-4155-9115-ad3edc39a8d0","name":"long-running-5s9vc-head"}
{"level":"info","ts":"2026-06-05T09:41:34.706Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"13826c98-17fd-421e-be34-65d272e7625f"}
{"level":"info","ts":"2026-06-05T09:41:34.706Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"13826c98-17fd-421e-be34-65d272e7625f"}
{"level":"info","ts":"2026-06-05T09:41:34.706Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"13826c98-17fd-421e-be34-65d272e7625f","1 head service found":"long-running-5s9vc-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:34.706Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"13826c98-17fd-421e-be34-65d272e7625f","Found 1 head Pod":"long-running-5s9vc-head-r7bq6","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:41:34.706Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"13826c98-17fd-421e-be34-65d272e7625f","head Pod":"long-running-5s9vc-head-r7bq6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5s9vc-head-r7bq6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:34.706Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"13826c98-17fd-421e-be34-65d272e7625f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:41:34.706Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"13826c98-17fd-421e-be34-65d272e7625f","worker Pod":"long-running-5s9vc-small-group-worker-pldls","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-5s9vc-small-group-worker-pldls. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:34.706Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"13826c98-17fd-421e-be34-65d272e7625f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:34.706Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"13826c98-17fd-421e-be34-65d272e7625f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:41:34.706Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"13826c98-17fd-421e-be34-65d272e7625f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:34.706Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"13826c98-17fd-421e-be34-65d272e7625f","podName":"long-running-5s9vc-head-r7bq6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:41:34.706Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"13826c98-17fd-421e-be34-65d272e7625f","podName":"long-running-5s9vc-head-r7bq6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:41:34.706Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"13826c98-17fd-421e-be34-65d272e7625f","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:41:32Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:41:32Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-05T09:41:34.706Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"13826c98-17fd-421e-be34-65d272e7625f","name":"long-running-5s9vc","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:41:34Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-5s9vc-head-r7bq6","serviceName":"long-running-5s9vc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:41:32Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:41:34Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-5s9vc-head-r7bq6","serviceName":"long-running-5s9vc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:41:32Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:41:34.710Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"686f90be-56c4-4155-9115-ad3edc39a8d0","name":"long-running-5s9vc-workers"}
{"level":"info","ts":"2026-06-05T09:41:34.710Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"686f90be-56c4-4155-9115-ad3edc39a8d0","cluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:34.718Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"06ad3992-e783-4f49-80a4-5097ba4bcc38"}
{"level":"info","ts":"2026-06-05T09:41:34.718Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"06ad3992-e783-4f49-80a4-5097ba4bcc38"}
{"level":"info","ts":"2026-06-05T09:41:34.718Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"06ad3992-e783-4f49-80a4-5097ba4bcc38","1 head service found":"long-running-5s9vc-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:34.718Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"cc84f55b-18bd-4619-a80a-ce8893a9c184","cluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:34.718Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"06ad3992-e783-4f49-80a4-5097ba4bcc38","Found 1 head Pod":"long-running-5s9vc-head-r7bq6","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:41:34.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"06ad3992-e783-4f49-80a4-5097ba4bcc38","head Pod":"long-running-5s9vc-head-r7bq6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5s9vc-head-r7bq6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:34.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"06ad3992-e783-4f49-80a4-5097ba4bcc38","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:41:34.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"06ad3992-e783-4f49-80a4-5097ba4bcc38","worker Pod":"long-running-5s9vc-small-group-worker-pldls","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-5s9vc-small-group-worker-pldls. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:34.719Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"83216491-2d19-41e5-96b7-13be7c8f2c7b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:34.718Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"88d4260c-d465-4373-aa26-4533d5bc6a95","rayCluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:34.719Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"83216491-2d19-41e5-96b7-13be7c8f2c7b","name":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:34.719Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"83216491-2d19-41e5-96b7-13be7c8f2c7b","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:34.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"06ad3992-e783-4f49-80a4-5097ba4bcc38","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:34.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"06ad3992-e783-4f49-80a4-5097ba4bcc38","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:41:34.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"06ad3992-e783-4f49-80a4-5097ba4bcc38","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:34.719Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"88d4260c-d465-4373-aa26-4533d5bc6a95","rayCluster":"long-running-5s9vc","podIPs":[]}
{"level":"info","ts":"2026-06-05T09:41:34.719Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"83216491-2d19-41e5-96b7-13be7c8f2c7b"}
{"level":"info","ts":"2026-06-05T09:41:34.719Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"83216491-2d19-41e5-96b7-13be7c8f2c7b","RayCluster":"long-running-5s9vc","State":""}
{"level":"info","ts":"2026-06-05T09:41:34.719Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"06ad3992-e783-4f49-80a4-5097ba4bcc38","podName":"long-running-5s9vc-small-group-worker-pldls","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:41:34.719Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"06ad3992-e783-4f49-80a4-5097ba4bcc38","podName":"long-running-5s9vc-small-group-worker-pldls","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:41:34.719Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"06ad3992-e783-4f49-80a4-5097ba4bcc38","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:41:32Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:41:32Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-05T09:41:34.719Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"06ad3992-e783-4f49-80a4-5097ba4bcc38","name":"long-running-5s9vc","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:41:34Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-5s9vc-head-r7bq6","serviceName":"long-running-5s9vc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:41:32Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:41:34Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-5s9vc-head-r7bq6","serviceName":"long-running-5s9vc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:41:32Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:41:34.725Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"06ad3992-e783-4f49-80a4-5097ba4bcc38","name":"long-running-5s9vc","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-5s9vc\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-t5s4r","name":"long-running-5s9vc"}}
{"level":"info","ts":"2026-06-05T09:41:34.725Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"06ad3992-e783-4f49-80a4-5097ba4bcc38"}
{"level":"error","ts":"2026-06-05T09:41:34.725Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"06ad3992-e783-4f49-80a4-5097ba4bcc38","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-5s9vc\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:41:34.728Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"cc84f55b-18bd-4619-a80a-ce8893a9c184","name":"long-running-5s9vc-head"}
{"level":"info","ts":"2026-06-05T09:41:34.731Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ec4ab5ae-935b-4617-ad39-64a7f51403e3"}
{"level":"info","ts":"2026-06-05T09:41:34.731Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ec4ab5ae-935b-4617-ad39-64a7f51403e3"}
{"level":"info","ts":"2026-06-05T09:41:34.732Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ec4ab5ae-935b-4617-ad39-64a7f51403e3","1 head service found":"long-running-5s9vc-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:34.805Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ec4ab5ae-935b-4617-ad39-64a7f51403e3","Found 1 head Pod":"long-running-5s9vc-head-r7bq6","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:41:34.806Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ec4ab5ae-935b-4617-ad39-64a7f51403e3","head Pod":"long-running-5s9vc-head-r7bq6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5s9vc-head-r7bq6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:34.806Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ec4ab5ae-935b-4617-ad39-64a7f51403e3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:41:34.806Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ec4ab5ae-935b-4617-ad39-64a7f51403e3","worker Pod":"long-running-5s9vc-small-group-worker-pldls","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-5s9vc-small-group-worker-pldls. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:34.806Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ec4ab5ae-935b-4617-ad39-64a7f51403e3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:34.806Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ec4ab5ae-935b-4617-ad39-64a7f51403e3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:41:34.806Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ec4ab5ae-935b-4617-ad39-64a7f51403e3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:34.806Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ec4ab5ae-935b-4617-ad39-64a7f51403e3","podName":"long-running-5s9vc-head-r7bq6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:41:34.806Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ec4ab5ae-935b-4617-ad39-64a7f51403e3","podName":"long-running-5s9vc-head-r7bq6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:41:34.806Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ec4ab5ae-935b-4617-ad39-64a7f51403e3","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:41:34.811Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"cc84f55b-18bd-4619-a80a-ce8893a9c184","name":"long-running-5s9vc-workers"}
{"level":"info","ts":"2026-06-05T09:41:34.811Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"cc84f55b-18bd-4619-a80a-ce8893a9c184","cluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:34.806Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ec4ab5ae-935b-4617-ad39-64a7f51403e3","seconds":300}
{"level":"info","ts":"2026-06-05T09:41:35.076Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c05a2960-cbe5-4d51-89f0-9de0bffbbe2d"}
{"level":"info","ts":"2026-06-05T09:41:35.076Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c05a2960-cbe5-4d51-89f0-9de0bffbbe2d"}
{"level":"info","ts":"2026-06-05T09:41:35.076Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c05a2960-cbe5-4d51-89f0-9de0bffbbe2d","1 head service found":"long-running-5s9vc-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:35.077Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c05a2960-cbe5-4d51-89f0-9de0bffbbe2d","Found 1 head Pod":"long-running-5s9vc-head-r7bq6","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:41:35.077Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c05a2960-cbe5-4d51-89f0-9de0bffbbe2d","head Pod":"long-running-5s9vc-head-r7bq6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5s9vc-head-r7bq6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:35.077Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c05a2960-cbe5-4d51-89f0-9de0bffbbe2d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:41:35.077Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c05a2960-cbe5-4d51-89f0-9de0bffbbe2d","worker Pod":"long-running-5s9vc-small-group-worker-pldls","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-5s9vc-small-group-worker-pldls. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:35.077Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c05a2960-cbe5-4d51-89f0-9de0bffbbe2d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:35.077Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c05a2960-cbe5-4d51-89f0-9de0bffbbe2d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:41:35.077Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c05a2960-cbe5-4d51-89f0-9de0bffbbe2d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:35.077Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c05a2960-cbe5-4d51-89f0-9de0bffbbe2d","podName":"long-running-5s9vc-head-r7bq6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:41:35.077Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c05a2960-cbe5-4d51-89f0-9de0bffbbe2d","podName":"long-running-5s9vc-head-r7bq6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:41:35.077Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c05a2960-cbe5-4d51-89f0-9de0bffbbe2d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:41:35.077Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c05a2960-cbe5-4d51-89f0-9de0bffbbe2d","seconds":300}
{"level":"info","ts":"2026-06-05T09:41:35.108Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c83bba8f-8ce7-44f0-bfd7-bf8a899e6a7f"}
{"level":"info","ts":"2026-06-05T09:41:35.108Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c83bba8f-8ce7-44f0-bfd7-bf8a899e6a7f"}
{"level":"info","ts":"2026-06-05T09:41:35.108Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c83bba8f-8ce7-44f0-bfd7-bf8a899e6a7f","1 head service found":"long-running-5s9vc-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:35.109Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c83bba8f-8ce7-44f0-bfd7-bf8a899e6a7f","Found 1 head Pod":"long-running-5s9vc-head-r7bq6","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:41:35.109Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c83bba8f-8ce7-44f0-bfd7-bf8a899e6a7f","head Pod":"long-running-5s9vc-head-r7bq6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5s9vc-head-r7bq6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:35.109Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c83bba8f-8ce7-44f0-bfd7-bf8a899e6a7f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:41:35.109Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c83bba8f-8ce7-44f0-bfd7-bf8a899e6a7f","worker Pod":"long-running-5s9vc-small-group-worker-pldls","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-5s9vc-small-group-worker-pldls. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:35.109Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c83bba8f-8ce7-44f0-bfd7-bf8a899e6a7f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:35.109Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c83bba8f-8ce7-44f0-bfd7-bf8a899e6a7f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:41:35.109Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c83bba8f-8ce7-44f0-bfd7-bf8a899e6a7f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:35.109Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c83bba8f-8ce7-44f0-bfd7-bf8a899e6a7f","podName":"long-running-5s9vc-head-r7bq6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:41:35.109Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c83bba8f-8ce7-44f0-bfd7-bf8a899e6a7f","podName":"long-running-5s9vc-head-r7bq6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:41:35.109Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c83bba8f-8ce7-44f0-bfd7-bf8a899e6a7f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:41:35.109Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c83bba8f-8ce7-44f0-bfd7-bf8a899e6a7f","seconds":300}
{"level":"info","ts":"2026-06-05T09:41:35.260Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"1d21bfe6-3fd2-4e32-89f1-5b747aa9f00d"}
{"level":"info","ts":"2026-06-05T09:41:35.260Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"1d21bfe6-3fd2-4e32-89f1-5b747aa9f00d"}
{"level":"info","ts":"2026-06-05T09:41:35.260Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"1d21bfe6-3fd2-4e32-89f1-5b747aa9f00d","1 head service found":"long-running-5s9vc-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:35.260Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"1d21bfe6-3fd2-4e32-89f1-5b747aa9f00d","Found 1 head Pod":"long-running-5s9vc-head-r7bq6","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:41:35.260Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"1d21bfe6-3fd2-4e32-89f1-5b747aa9f00d","head Pod":"long-running-5s9vc-head-r7bq6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5s9vc-head-r7bq6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:35.260Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"1d21bfe6-3fd2-4e32-89f1-5b747aa9f00d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:41:35.260Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"1d21bfe6-3fd2-4e32-89f1-5b747aa9f00d","worker Pod":"long-running-5s9vc-small-group-worker-pldls","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-5s9vc-small-group-worker-pldls. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:35.260Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"1d21bfe6-3fd2-4e32-89f1-5b747aa9f00d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:35.260Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"1d21bfe6-3fd2-4e32-89f1-5b747aa9f00d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:41:35.260Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"1d21bfe6-3fd2-4e32-89f1-5b747aa9f00d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:35.261Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"1d21bfe6-3fd2-4e32-89f1-5b747aa9f00d","podName":"long-running-5s9vc-small-group-worker-pldls","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:41:35.261Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"1d21bfe6-3fd2-4e32-89f1-5b747aa9f00d","podName":"long-running-5s9vc-small-group-worker-pldls","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:41:35.261Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"1d21bfe6-3fd2-4e32-89f1-5b747aa9f00d","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"long-running-5s9vc-head-r7bq6","serviceName":"long-running-5s9vc-head-svc"},"newHead":{"podIP":"10.132.0.20","serviceIP":"10.132.0.20","podName":"long-running-5s9vc-head-r7bq6","serviceName":"long-running-5s9vc-head-svc"}}
{"level":"info","ts":"2026-06-05T09:41:35.261Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"1d21bfe6-3fd2-4e32-89f1-5b747aa9f00d","name":"long-running-5s9vc","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:41:34Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-5s9vc-head-r7bq6","serviceName":"long-running-5s9vc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:41:32Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:41:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.20","serviceIP":"10.132.0.20","podName":"long-running-5s9vc-head-r7bq6","serviceName":"long-running-5s9vc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:41:32Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:41:35.273Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"8b691324-18ed-4f9e-85a0-d099cbe71352","cluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:35.273Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"87e30b3b-8d9f-4470-8e82-437d469b7609","rayCluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:35.273Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"87e30b3b-8d9f-4470-8e82-437d469b7609","pod":"long-running-5s9vc-head-r7bq6","ip":"10.132.0.20"}
{"level":"info","ts":"2026-06-05T09:41:35.273Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"14eaac7b-3dca-4a0e-a001-9c44c90a74ae","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:35.273Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"14eaac7b-3dca-4a0e-a001-9c44c90a74ae","name":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:35.273Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"14eaac7b-3dca-4a0e-a001-9c44c90a74ae","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:35.273Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"14eaac7b-3dca-4a0e-a001-9c44c90a74ae"}
{"level":"info","ts":"2026-06-05T09:41:35.273Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"14eaac7b-3dca-4a0e-a001-9c44c90a74ae","RayCluster":"long-running-5s9vc","State":""}
{"level":"info","ts":"2026-06-05T09:41:35.284Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"8b691324-18ed-4f9e-85a0-d099cbe71352","name":"long-running-5s9vc-head"}
{"level":"info","ts":"2026-06-05T09:41:35.285Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"87e30b3b-8d9f-4470-8e82-437d469b7609","certificate":"ray-head-cert-long-running-5s9vc","dnsNames":["long-running-5s9vc-head-svc","localhost","long-running-5s9vc-head-svc.test-ns-t5s4r.svc","long-running-5s9vc-head-svc.test-ns-t5s4r.svc.cluster.local"],"ipAddresses":["10.132.0.20","127.0.0.1"]}
{"level":"info","ts":"2026-06-05T09:41:35.297Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"8b691324-18ed-4f9e-85a0-d099cbe71352","name":"long-running-5s9vc-workers"}
{"level":"info","ts":"2026-06-05T09:41:35.297Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"8b691324-18ed-4f9e-85a0-d099cbe71352","cluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:35.298Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"87e30b3b-8d9f-4470-8e82-437d469b7609","certificate":"ray-worker-cert-long-running-5s9vc","dnsNames":["long-running-5s9vc-worker-svc","localhost","long-running-5s9vc-small-group","long-running-5s9vc-worker-svc.test-ns-t5s4r.svc","long-running-5s9vc-worker-svc.test-ns-t5s4r.svc.cluster.local","*.long-running-5s9vc-worker-svc.test-ns-t5s4r.svc","*.long-running-5s9vc-worker-svc.test-ns-t5s4r.svc.cluster.local","*-worker-*.test-ns-t5s4r.svc","*-worker-*.test-ns-t5s4r.svc.cluster.local"],"ipAddresses":["10.132.0.20","127.0.0.1"]}
{"level":"info","ts":"2026-06-05T09:41:35.298Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"87e30b3b-8d9f-4470-8e82-437d469b7609","rayCluster":"long-running-5s9vc","podIPs":["10.132.0.20"]}
{"level":"info","ts":"2026-06-05T09:41:35.826Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"f581b196-fb85-4948-bf3b-ecc16cd860d1"}
{"level":"info","ts":"2026-06-05T09:41:35.826Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"f581b196-fb85-4948-bf3b-ecc16cd860d1"}
{"level":"info","ts":"2026-06-05T09:41:35.826Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"f581b196-fb85-4948-bf3b-ecc16cd860d1","1 head service found":"long-running-5s9vc-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:35.826Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"f581b196-fb85-4948-bf3b-ecc16cd860d1","Found 1 head Pod":"long-running-5s9vc-head-r7bq6","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:41:35.826Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"f581b196-fb85-4948-bf3b-ecc16cd860d1","head Pod":"long-running-5s9vc-head-r7bq6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5s9vc-head-r7bq6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:35.826Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"f581b196-fb85-4948-bf3b-ecc16cd860d1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:41:35.826Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"f581b196-fb85-4948-bf3b-ecc16cd860d1","worker Pod":"long-running-5s9vc-small-group-worker-pldls","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-5s9vc-small-group-worker-pldls. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:35.826Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"f581b196-fb85-4948-bf3b-ecc16cd860d1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:35.826Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"f581b196-fb85-4948-bf3b-ecc16cd860d1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:41:35.826Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"f581b196-fb85-4948-bf3b-ecc16cd860d1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:35.826Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"f581b196-fb85-4948-bf3b-ecc16cd860d1","podName":"long-running-5s9vc-head-r7bq6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:41:35.827Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"f581b196-fb85-4948-bf3b-ecc16cd860d1","podName":"long-running-5s9vc-head-r7bq6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:41:35.827Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"f581b196-fb85-4948-bf3b-ecc16cd860d1","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:41:35.827Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"f581b196-fb85-4948-bf3b-ecc16cd860d1","seconds":300}
{"level":"info","ts":"2026-06-05T09:41:36.549Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"398e6a8e-2b49-4434-957e-b54423b8cbb3","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:41:36.549Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"398e6a8e-2b49-4434-957e-b54423b8cbb3","pod":"raycluster-head-vjd7d","ip":"10.134.0.20"}
{"level":"info","ts":"2026-06-05T09:41:36.549Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"398e6a8e-2b49-4434-957e-b54423b8cbb3","pod":"raycluster-small-group-worker-j9pbw","ip":"10.133.0.35"}
{"level":"info","ts":"2026-06-05T09:41:36.549Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"398e6a8e-2b49-4434-957e-b54423b8cbb3","rayCluster":"raycluster","podIPs":["10.134.0.20","10.133.0.35"]}
{"level":"info","ts":"2026-06-05T09:41:36.948Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"6c38810d-215f-4538-8f17-2c2e1d25e723","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:36.949Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"6c38810d-215f-4538-8f17-2c2e1d25e723","name":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:36.949Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"6c38810d-215f-4538-8f17-2c2e1d25e723","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:36.949Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"6c38810d-215f-4538-8f17-2c2e1d25e723"}
{"level":"info","ts":"2026-06-05T09:41:36.949Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"6c38810d-215f-4538-8f17-2c2e1d25e723","RayCluster":"long-running-5s9vc","State":""}
{"level":"info","ts":"2026-06-05T09:41:37.273Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"597be9d9-30e6-4d0a-93c7-4652c181e769"}
{"level":"info","ts":"2026-06-05T09:41:37.273Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"597be9d9-30e6-4d0a-93c7-4652c181e769"}
{"level":"info","ts":"2026-06-05T09:41:37.273Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"597be9d9-30e6-4d0a-93c7-4652c181e769","1 head service found":"long-running-5s9vc-head-svc"}
{"level":"info","ts":"2026-06-05T09:41:37.273Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"597be9d9-30e6-4d0a-93c7-4652c181e769","Found 1 head Pod":"long-running-5s9vc-head-r7bq6","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:41:37.273Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"597be9d9-30e6-4d0a-93c7-4652c181e769","head Pod":"long-running-5s9vc-head-r7bq6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5s9vc-head-r7bq6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:37.273Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"597be9d9-30e6-4d0a-93c7-4652c181e769","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:41:37.273Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"597be9d9-30e6-4d0a-93c7-4652c181e769","worker Pod":"long-running-5s9vc-small-group-worker-pldls","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-5s9vc-small-group-worker-pldls. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:41:37.273Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"597be9d9-30e6-4d0a-93c7-4652c181e769","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:37.273Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"597be9d9-30e6-4d0a-93c7-4652c181e769","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:41:37.273Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"597be9d9-30e6-4d0a-93c7-4652c181e769","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:41:37.273Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"597be9d9-30e6-4d0a-93c7-4652c181e769","podName":"long-running-5s9vc-head-r7bq6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:41:37.273Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"597be9d9-30e6-4d0a-93c7-4652c181e769","podName":"long-running-5s9vc-head-r7bq6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:41:37.273Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"597be9d9-30e6-4d0a-93c7-4652c181e769","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:41:37.273Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"597be9d9-30e6-4d0a-93c7-4652c181e769","seconds":300}
{"level":"info","ts":"2026-06-05T09:41:39.949Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"4ab25284-9f25-4c1a-813f-cbae7069afcd","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:39.949Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"4ab25284-9f25-4c1a-813f-cbae7069afcd","name":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:39.949Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"4ab25284-9f25-4c1a-813f-cbae7069afcd","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:39.949Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"4ab25284-9f25-4c1a-813f-cbae7069afcd"}
{"level":"info","ts":"2026-06-05T09:41:39.949Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"4ab25284-9f25-4c1a-813f-cbae7069afcd","RayCluster":"long-running-5s9vc","State":""}
{"level":"info","ts":"2026-06-05T09:41:42.950Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"a68a90e6-1311-4f63-aa0f-0e934fe1fd48","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:42.950Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"a68a90e6-1311-4f63-aa0f-0e934fe1fd48","name":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:42.950Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"a68a90e6-1311-4f63-aa0f-0e934fe1fd48","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:42.950Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"a68a90e6-1311-4f63-aa0f-0e934fe1fd48"}
{"level":"info","ts":"2026-06-05T09:41:42.950Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"a68a90e6-1311-4f63-aa0f-0e934fe1fd48","RayCluster":"long-running-5s9vc","State":""}
{"level":"info","ts":"2026-06-05T09:41:45.951Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"24c45cb2-6b81-420a-b34b-9db7d89b0eec","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:45.951Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"24c45cb2-6b81-420a-b34b-9db7d89b0eec","name":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:45.951Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"24c45cb2-6b81-420a-b34b-9db7d89b0eec","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:45.951Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"24c45cb2-6b81-420a-b34b-9db7d89b0eec"}
{"level":"info","ts":"2026-06-05T09:41:45.951Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"24c45cb2-6b81-420a-b34b-9db7d89b0eec","RayCluster":"long-running-5s9vc","State":""}
{"level":"info","ts":"2026-06-05T09:41:48.952Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"6d7a13ce-8af5-4eaf-b2b1-6b577bdd5a37","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:48.952Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"6d7a13ce-8af5-4eaf-b2b1-6b577bdd5a37","name":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:48.952Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"6d7a13ce-8af5-4eaf-b2b1-6b577bdd5a37","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:48.952Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"6d7a13ce-8af5-4eaf-b2b1-6b577bdd5a37"}
{"level":"info","ts":"2026-06-05T09:41:48.952Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"6d7a13ce-8af5-4eaf-b2b1-6b577bdd5a37","RayCluster":"long-running-5s9vc","State":""}
{"level":"info","ts":"2026-06-05T09:41:51.953Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"df67b1f6-4628-4dbd-8a6a-adfea9043524","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:51.953Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"df67b1f6-4628-4dbd-8a6a-adfea9043524","name":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:51.953Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"df67b1f6-4628-4dbd-8a6a-adfea9043524","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:51.953Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"df67b1f6-4628-4dbd-8a6a-adfea9043524"}
{"level":"info","ts":"2026-06-05T09:41:51.953Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"df67b1f6-4628-4dbd-8a6a-adfea9043524","RayCluster":"long-running-5s9vc","State":""}
{"level":"info","ts":"2026-06-05T09:41:54.954Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"f2095a89-82b9-40fd-9ac9-aa8b456747bd","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:54.954Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"f2095a89-82b9-40fd-9ac9-aa8b456747bd","name":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:54.954Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"f2095a89-82b9-40fd-9ac9-aa8b456747bd","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:54.954Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"f2095a89-82b9-40fd-9ac9-aa8b456747bd"}
{"level":"info","ts":"2026-06-05T09:41:54.954Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"f2095a89-82b9-40fd-9ac9-aa8b456747bd","RayCluster":"long-running-5s9vc","State":""}
{"level":"info","ts":"2026-06-05T09:41:57.955Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"fec0301f-1e07-4214-afb6-21e447604b79","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:41:57.955Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"fec0301f-1e07-4214-afb6-21e447604b79","name":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:41:57.955Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"fec0301f-1e07-4214-afb6-21e447604b79","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:41:57.955Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"fec0301f-1e07-4214-afb6-21e447604b79"}
{"level":"info","ts":"2026-06-05T09:41:57.955Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"fec0301f-1e07-4214-afb6-21e447604b79","RayCluster":"long-running-5s9vc","State":""}
{"level":"info","ts":"2026-06-05T09:42:00.955Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"ec66a09d-77e8-42ea-a29a-3a664c57e94c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:42:00.955Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"ec66a09d-77e8-42ea-a29a-3a664c57e94c","name":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:42:00.955Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"ec66a09d-77e8-42ea-a29a-3a664c57e94c","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:42:00.956Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"ec66a09d-77e8-42ea-a29a-3a664c57e94c"}
{"level":"info","ts":"2026-06-05T09:42:00.956Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"ec66a09d-77e8-42ea-a29a-3a664c57e94c","RayCluster":"long-running-5s9vc","State":""}
{"level":"info","ts":"2026-06-05T09:42:02.112Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"7e2978ce-63ab-4d27-a15a-44ee2f810b3e","rayCluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:42:02.112Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"7e2978ce-63ab-4d27-a15a-44ee2f810b3e","pod":"long-running-5s9vc-head-r7bq6","ip":"10.132.0.20"}
{"level":"info","ts":"2026-06-05T09:42:02.112Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"7e2978ce-63ab-4d27-a15a-44ee2f810b3e","pod":"long-running-5s9vc-small-group-worker-pldls","ip":"10.133.0.39"}
{"level":"info","ts":"2026-06-05T09:42:02.126Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"7e2978ce-63ab-4d27-a15a-44ee2f810b3e","certificate":"ray-head-cert-long-running-5s9vc","dnsNames":["long-running-5s9vc-head-svc","localhost","long-running-5s9vc-head-svc.test-ns-t5s4r.svc","long-running-5s9vc-head-svc.test-ns-t5s4r.svc.cluster.local"],"ipAddresses":["10.132.0.20","10.133.0.39","127.0.0.1"]}
{"level":"info","ts":"2026-06-05T09:42:02.142Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"7e2978ce-63ab-4d27-a15a-44ee2f810b3e","certificate":"ray-worker-cert-long-running-5s9vc","dnsNames":["long-running-5s9vc-worker-svc","localhost","long-running-5s9vc-small-group","long-running-5s9vc-worker-svc.test-ns-t5s4r.svc","long-running-5s9vc-worker-svc.test-ns-t5s4r.svc.cluster.local","*.long-running-5s9vc-worker-svc.test-ns-t5s4r.svc","*.long-running-5s9vc-worker-svc.test-ns-t5s4r.svc.cluster.local","*-worker-*.test-ns-t5s4r.svc","*-worker-*.test-ns-t5s4r.svc.cluster.local"],"ipAddresses":["10.132.0.20","10.133.0.39","127.0.0.1"]}
{"level":"info","ts":"2026-06-05T09:42:02.142Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"7e2978ce-63ab-4d27-a15a-44ee2f810b3e","rayCluster":"long-running-5s9vc","podIPs":["10.132.0.20","10.133.0.39"]}
{"level":"info","ts":"2026-06-05T09:42:03.956Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"a108acad-a54c-47ba-8831-e468f6bbc459","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:42:03.957Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"a108acad-a54c-47ba-8831-e468f6bbc459","name":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:42:03.957Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"a108acad-a54c-47ba-8831-e468f6bbc459","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:42:03.957Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"a108acad-a54c-47ba-8831-e468f6bbc459"}
{"level":"info","ts":"2026-06-05T09:42:03.957Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"a108acad-a54c-47ba-8831-e468f6bbc459","RayCluster":"long-running-5s9vc","State":""}
{"level":"info","ts":"2026-06-05T09:42:06.957Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"44a2077a-f9c2-44ba-a936-c3f009a168ef","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:42:06.957Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"44a2077a-f9c2-44ba-a936-c3f009a168ef","name":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:42:06.957Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"44a2077a-f9c2-44ba-a936-c3f009a168ef","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:42:06.957Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"44a2077a-f9c2-44ba-a936-c3f009a168ef"}
{"level":"info","ts":"2026-06-05T09:42:06.957Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"44a2077a-f9c2-44ba-a936-c3f009a168ef","RayCluster":"long-running-5s9vc","State":""}
{"level":"info","ts":"2026-06-05T09:42:09.959Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"6bf6917d-05db-40ac-be92-71aa3cf1c991","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:42:09.959Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"6bf6917d-05db-40ac-be92-71aa3cf1c991","name":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:42:09.959Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"6bf6917d-05db-40ac-be92-71aa3cf1c991","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:42:09.959Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"6bf6917d-05db-40ac-be92-71aa3cf1c991"}
{"level":"info","ts":"2026-06-05T09:42:09.959Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"6bf6917d-05db-40ac-be92-71aa3cf1c991","RayCluster":"long-running-5s9vc","State":""}
{"level":"info","ts":"2026-06-05T09:42:12.959Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"73126858-5753-452e-93c8-6dd814435273","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:42:12.959Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"73126858-5753-452e-93c8-6dd814435273","name":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:42:12.959Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"73126858-5753-452e-93c8-6dd814435273","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:42:12.960Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"73126858-5753-452e-93c8-6dd814435273"}
{"level":"info","ts":"2026-06-05T09:42:12.960Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"73126858-5753-452e-93c8-6dd814435273","RayCluster":"long-running-5s9vc","State":""}
{"level":"info","ts":"2026-06-05T09:42:15.960Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"482d77c6-7d25-419c-ad3b-037b4edc487d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:42:15.960Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"482d77c6-7d25-419c-ad3b-037b4edc487d","name":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:42:15.960Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"482d77c6-7d25-419c-ad3b-037b4edc487d","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:42:15.961Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"482d77c6-7d25-419c-ad3b-037b4edc487d"}
{"level":"info","ts":"2026-06-05T09:42:15.961Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"482d77c6-7d25-419c-ad3b-037b4edc487d","RayCluster":"long-running-5s9vc","State":""}
{"level":"info","ts":"2026-06-05T09:42:18.961Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"cbaf1d42-81ff-4879-a6f1-04a6e0b35474","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:42:18.961Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"cbaf1d42-81ff-4879-a6f1-04a6e0b35474","name":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:42:18.961Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"cbaf1d42-81ff-4879-a6f1-04a6e0b35474","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:42:18.961Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"cbaf1d42-81ff-4879-a6f1-04a6e0b35474"}
{"level":"info","ts":"2026-06-05T09:42:18.961Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"cbaf1d42-81ff-4879-a6f1-04a6e0b35474","RayCluster":"long-running-5s9vc","State":""}
{"level":"info","ts":"2026-06-05T09:42:21.962Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"a51892f6-d867-4520-84f1-cfac360ab038","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:42:21.962Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"a51892f6-d867-4520-84f1-cfac360ab038","name":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:42:21.962Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"a51892f6-d867-4520-84f1-cfac360ab038","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:42:21.962Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"a51892f6-d867-4520-84f1-cfac360ab038"}
{"level":"info","ts":"2026-06-05T09:42:21.962Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"a51892f6-d867-4520-84f1-cfac360ab038","RayCluster":"long-running-5s9vc","State":""}
{"level":"info","ts":"2026-06-05T09:42:24.963Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"85eb4bcd-b40d-4e57-a9fc-c34810e76e00","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:42:24.963Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"85eb4bcd-b40d-4e57-a9fc-c34810e76e00","name":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:42:24.963Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"85eb4bcd-b40d-4e57-a9fc-c34810e76e00","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:42:24.963Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"85eb4bcd-b40d-4e57-a9fc-c34810e76e00"}
{"level":"info","ts":"2026-06-05T09:42:24.963Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"85eb4bcd-b40d-4e57-a9fc-c34810e76e00","RayCluster":"long-running-5s9vc","State":""}
{"level":"info","ts":"2026-06-05T09:42:27.964Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"89e268a4-bb2c-4128-8733-8ab7cf68f75f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:42:27.964Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"89e268a4-bb2c-4128-8733-8ab7cf68f75f","name":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:42:27.964Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"89e268a4-bb2c-4128-8733-8ab7cf68f75f","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:42:27.964Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"89e268a4-bb2c-4128-8733-8ab7cf68f75f"}
{"level":"info","ts":"2026-06-05T09:42:27.964Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"89e268a4-bb2c-4128-8733-8ab7cf68f75f","RayCluster":"long-running-5s9vc","State":""}
{"level":"info","ts":"2026-06-05T09:42:30.965Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"ca70b694-0661-4896-a2dc-af60085c2919","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:42:30.965Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"ca70b694-0661-4896-a2dc-af60085c2919","name":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:42:30.965Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"ca70b694-0661-4896-a2dc-af60085c2919","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:42:30.965Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"ca70b694-0661-4896-a2dc-af60085c2919"}
{"level":"info","ts":"2026-06-05T09:42:30.965Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"ca70b694-0661-4896-a2dc-af60085c2919","RayCluster":"long-running-5s9vc","State":""}
{"level":"info","ts":"2026-06-05T09:42:33.966Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"aa39736b-86e3-42d5-8127-f7b11dc63e7d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:42:33.966Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"aa39736b-86e3-42d5-8127-f7b11dc63e7d","name":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:42:33.966Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"aa39736b-86e3-42d5-8127-f7b11dc63e7d","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:42:33.966Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"aa39736b-86e3-42d5-8127-f7b11dc63e7d"}
{"level":"info","ts":"2026-06-05T09:42:33.966Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"aa39736b-86e3-42d5-8127-f7b11dc63e7d","RayCluster":"long-running-5s9vc","State":""}
{"level":"info","ts":"2026-06-05T09:42:36.551Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"720ab8bd-b1df-44fb-b0f7-40c8b7118068","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:42:36.551Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"720ab8bd-b1df-44fb-b0f7-40c8b7118068","pod":"raycluster-head-vjd7d","ip":"10.134.0.20"}
{"level":"info","ts":"2026-06-05T09:42:36.551Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"720ab8bd-b1df-44fb-b0f7-40c8b7118068","pod":"raycluster-small-group-worker-j9pbw","ip":"10.133.0.35"}
{"level":"info","ts":"2026-06-05T09:42:36.551Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"720ab8bd-b1df-44fb-b0f7-40c8b7118068","rayCluster":"raycluster","podIPs":["10.134.0.20","10.133.0.35"]}
{"level":"info","ts":"2026-06-05T09:42:36.967Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"0ab7b34e-dc80-4710-94ad-799421080b3b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:42:36.967Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"0ab7b34e-dc80-4710-94ad-799421080b3b","name":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:42:36.967Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"0ab7b34e-dc80-4710-94ad-799421080b3b","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:42:36.967Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"0ab7b34e-dc80-4710-94ad-799421080b3b"}
{"level":"info","ts":"2026-06-05T09:42:36.967Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"0ab7b34e-dc80-4710-94ad-799421080b3b","RayCluster":"long-running-5s9vc","State":""}
{"level":"info","ts":"2026-06-05T09:42:39.968Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"37504f48-ad1a-4fc1-aae6-aca24fdf58a2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:42:39.968Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"37504f48-ad1a-4fc1-aae6-aca24fdf58a2","name":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:42:39.968Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"37504f48-ad1a-4fc1-aae6-aca24fdf58a2","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:42:39.968Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"37504f48-ad1a-4fc1-aae6-aca24fdf58a2"}
{"level":"info","ts":"2026-06-05T09:42:39.968Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"37504f48-ad1a-4fc1-aae6-aca24fdf58a2","RayCluster":"long-running-5s9vc","State":""}
{"level":"info","ts":"2026-06-05T09:42:42.969Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"9c458be3-f06e-4da5-a907-f7627fdd9992","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:42:42.969Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"9c458be3-f06e-4da5-a907-f7627fdd9992","name":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:42:42.969Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"9c458be3-f06e-4da5-a907-f7627fdd9992","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:42:42.969Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"9c458be3-f06e-4da5-a907-f7627fdd9992"}
{"level":"info","ts":"2026-06-05T09:42:42.969Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"9c458be3-f06e-4da5-a907-f7627fdd9992","RayCluster":"long-running-5s9vc","State":""}
{"level":"info","ts":"2026-06-05T09:42:45.491Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ccc23605-febd-4867-8d9d-1fb561ea34eb"}
{"level":"info","ts":"2026-06-05T09:42:45.491Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ccc23605-febd-4867-8d9d-1fb561ea34eb"}
{"level":"info","ts":"2026-06-05T09:42:45.492Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ccc23605-febd-4867-8d9d-1fb561ea34eb","1 head service found":"long-running-5s9vc-head-svc"}
{"level":"info","ts":"2026-06-05T09:42:45.492Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ccc23605-febd-4867-8d9d-1fb561ea34eb","Found 1 head Pod":"long-running-5s9vc-head-r7bq6","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:42:45.492Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ccc23605-febd-4867-8d9d-1fb561ea34eb","head Pod":"long-running-5s9vc-head-r7bq6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5s9vc-head-r7bq6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:42:45.492Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ccc23605-febd-4867-8d9d-1fb561ea34eb","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:42:45.492Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ccc23605-febd-4867-8d9d-1fb561ea34eb","worker Pod":"long-running-5s9vc-small-group-worker-pldls","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-5s9vc-small-group-worker-pldls. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:42:45.492Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ccc23605-febd-4867-8d9d-1fb561ea34eb","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:42:45.492Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ccc23605-febd-4867-8d9d-1fb561ea34eb","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:42:45.492Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ccc23605-febd-4867-8d9d-1fb561ea34eb","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:42:45.492Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ccc23605-febd-4867-8d9d-1fb561ea34eb","podName":"long-running-5s9vc-head-r7bq6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:42:45.492Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ccc23605-febd-4867-8d9d-1fb561ea34eb","podName":"long-running-5s9vc-head-r7bq6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:42:45.492Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ccc23605-febd-4867-8d9d-1fb561ea34eb","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:42:45.492Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"ccc23605-febd-4867-8d9d-1fb561ea34eb","seconds":300}
{"level":"info","ts":"2026-06-05T09:42:45.970Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"d272c98b-99ea-4182-96a7-d47599be91d0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:42:45.970Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"d272c98b-99ea-4182-96a7-d47599be91d0","name":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:42:45.970Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"d272c98b-99ea-4182-96a7-d47599be91d0","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:42:45.970Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"d272c98b-99ea-4182-96a7-d47599be91d0"}
{"level":"info","ts":"2026-06-05T09:42:45.970Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"d272c98b-99ea-4182-96a7-d47599be91d0","RayCluster":"long-running-5s9vc","State":""}
{"level":"info","ts":"2026-06-05T09:42:46.503Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"d703b346-13e5-4497-b3f3-e5d05482a78c"}
{"level":"info","ts":"2026-06-05T09:42:46.503Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"d703b346-13e5-4497-b3f3-e5d05482a78c"}
{"level":"info","ts":"2026-06-05T09:42:46.503Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"d703b346-13e5-4497-b3f3-e5d05482a78c","1 head service found":"long-running-5s9vc-head-svc"}
{"level":"info","ts":"2026-06-05T09:42:46.503Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"d703b346-13e5-4497-b3f3-e5d05482a78c","Found 1 head Pod":"long-running-5s9vc-head-r7bq6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:42:46.503Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"d703b346-13e5-4497-b3f3-e5d05482a78c","head Pod":"long-running-5s9vc-head-r7bq6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5s9vc-head-r7bq6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:42:46.503Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"d703b346-13e5-4497-b3f3-e5d05482a78c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:42:46.503Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"d703b346-13e5-4497-b3f3-e5d05482a78c","worker Pod":"long-running-5s9vc-small-group-worker-pldls","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-5s9vc-small-group-worker-pldls. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:42:46.503Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"d703b346-13e5-4497-b3f3-e5d05482a78c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:42:46.503Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"d703b346-13e5-4497-b3f3-e5d05482a78c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:42:46.503Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"d703b346-13e5-4497-b3f3-e5d05482a78c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:42:46.503Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"d703b346-13e5-4497-b3f3-e5d05482a78c","podName":"long-running-5s9vc-head-r7bq6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:41:34Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:42:46.503Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"d703b346-13e5-4497-b3f3-e5d05482a78c","podName":"long-running-5s9vc-head-r7bq6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:41:34Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:42:46.503Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"d703b346-13e5-4497-b3f3-e5d05482a78c","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:41:32Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:41:32Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-05T09:42:46.503Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"d703b346-13e5-4497-b3f3-e5d05482a78c","name":"long-running-5s9vc","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:41:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.20","serviceIP":"10.132.0.20","podName":"long-running-5s9vc-head-r7bq6","serviceName":"long-running-5s9vc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:41:32Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:42:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.20","serviceIP":"10.132.0.20","podName":"long-running-5s9vc-head-r7bq6","serviceName":"long-running-5s9vc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:41:32Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:42:46.517Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"4cd5e43b-a874-486c-ad09-9cdbe2cc534b","rayCluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:42:46.517Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"b00ababd-698e-4cc2-9372-17a51e2bc6ed","cluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:42:46.518Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"a4a428bb-7a8f-436c-a381-46f45aff9690","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:42:46.518Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"a4a428bb-7a8f-436c-a381-46f45aff9690","name":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:42:46.518Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"a4a428bb-7a8f-436c-a381-46f45aff9690","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:42:46.518Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"4cd5e43b-a874-486c-ad09-9cdbe2cc534b","pod":"long-running-5s9vc-small-group-worker-pldls","ip":"10.133.0.39"}
{"level":"info","ts":"2026-06-05T09:42:46.518Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"4cd5e43b-a874-486c-ad09-9cdbe2cc534b","pod":"long-running-5s9vc-head-r7bq6","ip":"10.132.0.20"}
{"level":"info","ts":"2026-06-05T09:42:46.518Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"4cd5e43b-a874-486c-ad09-9cdbe2cc534b","rayCluster":"long-running-5s9vc","podIPs":["10.133.0.39","10.132.0.20"]}
{"level":"info","ts":"2026-06-05T09:42:46.518Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"a4a428bb-7a8f-436c-a381-46f45aff9690"}
{"level":"info","ts":"2026-06-05T09:42:46.518Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"a4a428bb-7a8f-436c-a381-46f45aff9690","RayCluster":"long-running-5s9vc","State":""}
{"level":"info","ts":"2026-06-05T09:42:46.536Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"b00ababd-698e-4cc2-9372-17a51e2bc6ed","name":"long-running-5s9vc-head"}
{"level":"info","ts":"2026-06-05T09:42:46.551Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"b00ababd-698e-4cc2-9372-17a51e2bc6ed","name":"long-running-5s9vc-workers"}
{"level":"info","ts":"2026-06-05T09:42:46.551Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"b00ababd-698e-4cc2-9372-17a51e2bc6ed","cluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:42:48.518Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c3d38a9f-4bd4-4c09-84aa-cc47b62ff4bb"}
{"level":"info","ts":"2026-06-05T09:42:48.518Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c3d38a9f-4bd4-4c09-84aa-cc47b62ff4bb"}
{"level":"info","ts":"2026-06-05T09:42:48.518Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c3d38a9f-4bd4-4c09-84aa-cc47b62ff4bb","1 head service found":"long-running-5s9vc-head-svc"}
{"level":"info","ts":"2026-06-05T09:42:48.518Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c3d38a9f-4bd4-4c09-84aa-cc47b62ff4bb","Found 1 head Pod":"long-running-5s9vc-head-r7bq6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:42:48.518Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c3d38a9f-4bd4-4c09-84aa-cc47b62ff4bb","head Pod":"long-running-5s9vc-head-r7bq6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5s9vc-head-r7bq6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:42:48.518Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c3d38a9f-4bd4-4c09-84aa-cc47b62ff4bb","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:42:48.518Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c3d38a9f-4bd4-4c09-84aa-cc47b62ff4bb","worker Pod":"long-running-5s9vc-small-group-worker-pldls","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-5s9vc-small-group-worker-pldls. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:42:48.518Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c3d38a9f-4bd4-4c09-84aa-cc47b62ff4bb","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:42:48.518Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c3d38a9f-4bd4-4c09-84aa-cc47b62ff4bb","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:42:48.518Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c3d38a9f-4bd4-4c09-84aa-cc47b62ff4bb","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:42:48.518Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c3d38a9f-4bd4-4c09-84aa-cc47b62ff4bb","podName":"long-running-5s9vc-head-r7bq6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:41:34Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:42:48.518Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c3d38a9f-4bd4-4c09-84aa-cc47b62ff4bb","podName":"long-running-5s9vc-head-r7bq6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:41:34Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:42:48.518Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c3d38a9f-4bd4-4c09-84aa-cc47b62ff4bb","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:42:48.518Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"c3d38a9f-4bd4-4c09-84aa-cc47b62ff4bb","seconds":300}
{"level":"info","ts":"2026-06-05T09:42:48.971Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"8934e438-193d-4e9e-ab27-af318aa59084","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:42:48.971Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"8934e438-193d-4e9e-ab27-af318aa59084","name":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:42:48.971Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"8934e438-193d-4e9e-ab27-af318aa59084","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:42:48.971Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"8934e438-193d-4e9e-ab27-af318aa59084"}
{"level":"info","ts":"2026-06-05T09:42:48.971Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"8934e438-193d-4e9e-ab27-af318aa59084","RayCluster":"long-running-5s9vc","State":""}
{"level":"info","ts":"2026-06-05T09:42:50.185Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a0873bc7-dcff-4c65-9b28-077809c74d21"}
{"level":"info","ts":"2026-06-05T09:42:50.186Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a0873bc7-dcff-4c65-9b28-077809c74d21"}
{"level":"info","ts":"2026-06-05T09:42:50.186Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a0873bc7-dcff-4c65-9b28-077809c74d21","1 head service found":"long-running-5s9vc-head-svc"}
{"level":"info","ts":"2026-06-05T09:42:50.186Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a0873bc7-dcff-4c65-9b28-077809c74d21","Found 1 head Pod":"long-running-5s9vc-head-r7bq6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:42:50.186Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a0873bc7-dcff-4c65-9b28-077809c74d21","head Pod":"long-running-5s9vc-head-r7bq6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5s9vc-head-r7bq6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:42:50.186Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a0873bc7-dcff-4c65-9b28-077809c74d21","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:42:50.186Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a0873bc7-dcff-4c65-9b28-077809c74d21","worker Pod":"long-running-5s9vc-small-group-worker-pldls","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-5s9vc-small-group-worker-pldls. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:42:50.186Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a0873bc7-dcff-4c65-9b28-077809c74d21","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:42:50.186Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a0873bc7-dcff-4c65-9b28-077809c74d21","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:42:50.186Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a0873bc7-dcff-4c65-9b28-077809c74d21","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:42:50.186Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a0873bc7-dcff-4c65-9b28-077809c74d21","podName":"long-running-5s9vc-head-r7bq6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:41:34Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:42:50.186Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a0873bc7-dcff-4c65-9b28-077809c74d21","podName":"long-running-5s9vc-head-r7bq6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:41:34Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:42:50.186Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a0873bc7-dcff-4c65-9b28-077809c74d21","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:42:50.186Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a0873bc7-dcff-4c65-9b28-077809c74d21","seconds":300}
{"level":"info","ts":"2026-06-05T09:42:51.189Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"71e89e38-d65f-4ce6-9f4b-576339dbe15c"}
{"level":"info","ts":"2026-06-05T09:42:51.189Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"71e89e38-d65f-4ce6-9f4b-576339dbe15c"}
{"level":"info","ts":"2026-06-05T09:42:51.189Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"71e89e38-d65f-4ce6-9f4b-576339dbe15c","1 head service found":"long-running-5s9vc-head-svc"}
{"level":"info","ts":"2026-06-05T09:42:51.189Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"71e89e38-d65f-4ce6-9f4b-576339dbe15c","Found 1 head Pod":"long-running-5s9vc-head-r7bq6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:42:51.189Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"71e89e38-d65f-4ce6-9f4b-576339dbe15c","head Pod":"long-running-5s9vc-head-r7bq6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5s9vc-head-r7bq6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:42:51.189Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"71e89e38-d65f-4ce6-9f4b-576339dbe15c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:42:51.189Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"71e89e38-d65f-4ce6-9f4b-576339dbe15c","worker Pod":"long-running-5s9vc-small-group-worker-pldls","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-5s9vc-small-group-worker-pldls. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:42:51.189Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"71e89e38-d65f-4ce6-9f4b-576339dbe15c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:42:51.189Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"71e89e38-d65f-4ce6-9f4b-576339dbe15c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:42:51.189Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"71e89e38-d65f-4ce6-9f4b-576339dbe15c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:42:51.189Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"71e89e38-d65f-4ce6-9f4b-576339dbe15c","podName":"long-running-5s9vc-head-r7bq6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:41:34Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:42:51.189Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"71e89e38-d65f-4ce6-9f4b-576339dbe15c","podName":"long-running-5s9vc-head-r7bq6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:41:34Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:42:51.189Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"71e89e38-d65f-4ce6-9f4b-576339dbe15c","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-05T09:42:51.189Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"71e89e38-d65f-4ce6-9f4b-576339dbe15c","name":"long-running-5s9vc","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:42:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.20","serviceIP":"10.132.0.20","podName":"long-running-5s9vc-head-r7bq6","serviceName":"long-running-5s9vc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:41:32Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:42:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.20","serviceIP":"10.132.0.20","podName":"long-running-5s9vc-head-r7bq6","serviceName":"long-running-5s9vc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:41:32Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:42:51.203Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"b23f77f9-e8ee-4958-b1b7-ea54662e8215","rayCluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:42:51.203Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"a8ef3504-d21e-4944-9c51-732a6e13bb40","cluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:42:51.203Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"bb4f49a3-967c-411e-8a21-d4d026421abb","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:42:51.203Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"bb4f49a3-967c-411e-8a21-d4d026421abb","name":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:42:51.203Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"bb4f49a3-967c-411e-8a21-d4d026421abb","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:42:51.203Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"b23f77f9-e8ee-4958-b1b7-ea54662e8215","pod":"long-running-5s9vc-head-r7bq6","ip":"10.132.0.20"}
{"level":"info","ts":"2026-06-05T09:42:51.203Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"b23f77f9-e8ee-4958-b1b7-ea54662e8215","pod":"long-running-5s9vc-small-group-worker-pldls","ip":"10.133.0.39"}
{"level":"info","ts":"2026-06-05T09:42:51.203Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"b23f77f9-e8ee-4958-b1b7-ea54662e8215","rayCluster":"long-running-5s9vc","podIPs":["10.132.0.20","10.133.0.39"]}
{"level":"info","ts":"2026-06-05T09:42:51.203Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"bb4f49a3-967c-411e-8a21-d4d026421abb"}
{"level":"info","ts":"2026-06-05T09:42:51.203Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"bb4f49a3-967c-411e-8a21-d4d026421abb","RayCluster":"long-running-5s9vc","State":""}
{"level":"info","ts":"2026-06-05T09:42:51.213Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"a8ef3504-d21e-4944-9c51-732a6e13bb40","name":"long-running-5s9vc-head"}
{"level":"info","ts":"2026-06-05T09:42:51.226Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"a8ef3504-d21e-4944-9c51-732a6e13bb40","name":"long-running-5s9vc-workers"}
{"level":"info","ts":"2026-06-05T09:42:51.226Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"a8ef3504-d21e-4944-9c51-732a6e13bb40","cluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:42:51.972Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"52c747a7-41c8-499c-8cf9-39e6b889e0be","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:42:51.972Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"52c747a7-41c8-499c-8cf9-39e6b889e0be","name":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:42:51.972Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"52c747a7-41c8-499c-8cf9-39e6b889e0be","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:42:51.972Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"52c747a7-41c8-499c-8cf9-39e6b889e0be"}
{"level":"info","ts":"2026-06-05T09:42:51.972Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"52c747a7-41c8-499c-8cf9-39e6b889e0be","RayCluster":"long-running-5s9vc","State":""}
{"level":"info","ts":"2026-06-05T09:42:53.203Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"87b25818-2d3e-4650-9538-962b7308322b"}
{"level":"info","ts":"2026-06-05T09:42:53.203Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"87b25818-2d3e-4650-9538-962b7308322b"}
{"level":"info","ts":"2026-06-05T09:42:53.203Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"87b25818-2d3e-4650-9538-962b7308322b","1 head service found":"long-running-5s9vc-head-svc"}
{"level":"info","ts":"2026-06-05T09:42:53.203Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"87b25818-2d3e-4650-9538-962b7308322b","Found 1 head Pod":"long-running-5s9vc-head-r7bq6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:42:53.203Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"87b25818-2d3e-4650-9538-962b7308322b","head Pod":"long-running-5s9vc-head-r7bq6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5s9vc-head-r7bq6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:42:53.203Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"87b25818-2d3e-4650-9538-962b7308322b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:42:53.203Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"87b25818-2d3e-4650-9538-962b7308322b","worker Pod":"long-running-5s9vc-small-group-worker-pldls","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-5s9vc-small-group-worker-pldls. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:42:53.203Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"87b25818-2d3e-4650-9538-962b7308322b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:42:53.203Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"87b25818-2d3e-4650-9538-962b7308322b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:42:53.203Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"87b25818-2d3e-4650-9538-962b7308322b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:42:53.203Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"87b25818-2d3e-4650-9538-962b7308322b","podName":"long-running-5s9vc-head-r7bq6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:41:34Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:42:53.203Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"87b25818-2d3e-4650-9538-962b7308322b","podName":"long-running-5s9vc-head-r7bq6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:41:34Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:42:53.203Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"87b25818-2d3e-4650-9538-962b7308322b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:42:53.203Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"87b25818-2d3e-4650-9538-962b7308322b","seconds":300}
{"level":"info","ts":"2026-06-05T09:42:54.973Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"dc650d23-f851-4703-85b2-5c681940f74e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:42:54.973Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"dc650d23-f851-4703-85b2-5c681940f74e","name":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:42:54.973Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"dc650d23-f851-4703-85b2-5c681940f74e","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:42:54.973Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"dc650d23-f851-4703-85b2-5c681940f74e"}
{"level":"info","ts":"2026-06-05T09:42:54.973Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"dc650d23-f851-4703-85b2-5c681940f74e","RayCluster":"long-running-5s9vc","State":""}
{"level":"info","ts":"2026-06-05T09:42:57.530Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a2bba409-8a53-4fd8-9650-933532c2b45b"}
{"level":"info","ts":"2026-06-05T09:42:57.530Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a2bba409-8a53-4fd8-9650-933532c2b45b"}
{"level":"info","ts":"2026-06-05T09:42:57.530Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a2bba409-8a53-4fd8-9650-933532c2b45b","1 head service found":"long-running-5s9vc-head-svc"}
{"level":"info","ts":"2026-06-05T09:42:57.530Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a2bba409-8a53-4fd8-9650-933532c2b45b","Found 1 head Pod":"long-running-5s9vc-head-r7bq6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:42:57.530Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a2bba409-8a53-4fd8-9650-933532c2b45b","head Pod":"long-running-5s9vc-head-r7bq6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5s9vc-head-r7bq6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:42:57.530Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a2bba409-8a53-4fd8-9650-933532c2b45b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:42:57.530Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a2bba409-8a53-4fd8-9650-933532c2b45b","worker Pod":"long-running-5s9vc-small-group-worker-pldls","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-5s9vc-small-group-worker-pldls. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:42:57.530Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a2bba409-8a53-4fd8-9650-933532c2b45b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:42:57.530Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a2bba409-8a53-4fd8-9650-933532c2b45b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:42:57.530Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a2bba409-8a53-4fd8-9650-933532c2b45b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:42:57.530Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a2bba409-8a53-4fd8-9650-933532c2b45b","podName":"long-running-5s9vc-small-group-worker-pldls","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:41:34Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-05T09:42:57.530Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a2bba409-8a53-4fd8-9650-933532c2b45b","podName":"long-running-5s9vc-small-group-worker-pldls","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:41:34Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-05T09:42:57.530Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a2bba409-8a53-4fd8-9650-933532c2b45b","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:41:32Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:42:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:41:32Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-05T09:42:57.530Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"a2bba409-8a53-4fd8-9650-933532c2b45b","name":"long-running-5s9vc","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:42:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.20","serviceIP":"10.132.0.20","podName":"long-running-5s9vc-head-r7bq6","serviceName":"long-running-5s9vc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:41:32Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:42:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.20","serviceIP":"10.132.0.20","podName":"long-running-5s9vc-head-r7bq6","serviceName":"long-running-5s9vc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:42:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:41:32Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:42:57.542Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"711ffdf1-3b81-4ffb-9050-5ca71e5ef1c3","rayCluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:42:57.542Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"688d92df-06d5-4e90-93f7-f609c10cfa0e","cluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:42:57.542Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"711ffdf1-3b81-4ffb-9050-5ca71e5ef1c3","pod":"long-running-5s9vc-head-r7bq6","ip":"10.132.0.20"}
{"level":"info","ts":"2026-06-05T09:42:57.542Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"711ffdf1-3b81-4ffb-9050-5ca71e5ef1c3","pod":"long-running-5s9vc-small-group-worker-pldls","ip":"10.133.0.39"}
{"level":"info","ts":"2026-06-05T09:42:57.542Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"92631cf5-4b15-449d-9e53-d17d85a766d7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:42:57.542Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"92631cf5-4b15-449d-9e53-d17d85a766d7","name":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:42:57.542Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"92631cf5-4b15-449d-9e53-d17d85a766d7","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:42:57.542Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"711ffdf1-3b81-4ffb-9050-5ca71e5ef1c3","rayCluster":"long-running-5s9vc","podIPs":["10.132.0.20","10.133.0.39"]}
{"level":"info","ts":"2026-06-05T09:42:57.543Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"92631cf5-4b15-449d-9e53-d17d85a766d7"}
{"level":"info","ts":"2026-06-05T09:42:57.543Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"92631cf5-4b15-449d-9e53-d17d85a766d7","RayCluster":"long-running-5s9vc","State":""}
{"level":"info","ts":"2026-06-05T09:42:57.552Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"688d92df-06d5-4e90-93f7-f609c10cfa0e","name":"long-running-5s9vc-head"}
{"level":"info","ts":"2026-06-05T09:42:57.562Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"688d92df-06d5-4e90-93f7-f609c10cfa0e","name":"long-running-5s9vc-workers"}
{"level":"info","ts":"2026-06-05T09:42:57.562Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"688d92df-06d5-4e90-93f7-f609c10cfa0e","cluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:42:57.973Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"f6c0a68c-2ab3-475d-a6ed-ada08bb218f2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:42:57.973Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"f6c0a68c-2ab3-475d-a6ed-ada08bb218f2","name":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:42:57.973Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"f6c0a68c-2ab3-475d-a6ed-ada08bb218f2","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:42:57.973Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"f6c0a68c-2ab3-475d-a6ed-ada08bb218f2"}
{"level":"info","ts":"2026-06-05T09:42:57.973Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"f6c0a68c-2ab3-475d-a6ed-ada08bb218f2","RayCluster":"long-running-5s9vc","State":""}
{"level":"info","ts":"2026-06-05T09:42:59.542Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"bffdb127-56c1-470d-ad22-49397ab41c60"}
{"level":"info","ts":"2026-06-05T09:42:59.542Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"bffdb127-56c1-470d-ad22-49397ab41c60"}
{"level":"info","ts":"2026-06-05T09:42:59.542Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"bffdb127-56c1-470d-ad22-49397ab41c60","1 head service found":"long-running-5s9vc-head-svc"}
{"level":"info","ts":"2026-06-05T09:42:59.542Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"bffdb127-56c1-470d-ad22-49397ab41c60","Found 1 head Pod":"long-running-5s9vc-head-r7bq6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:42:59.542Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"bffdb127-56c1-470d-ad22-49397ab41c60","head Pod":"long-running-5s9vc-head-r7bq6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5s9vc-head-r7bq6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:42:59.542Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"bffdb127-56c1-470d-ad22-49397ab41c60","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:42:59.542Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"bffdb127-56c1-470d-ad22-49397ab41c60","worker Pod":"long-running-5s9vc-small-group-worker-pldls","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-5s9vc-small-group-worker-pldls. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:42:59.542Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"bffdb127-56c1-470d-ad22-49397ab41c60","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:42:59.542Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"bffdb127-56c1-470d-ad22-49397ab41c60","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:42:59.542Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"bffdb127-56c1-470d-ad22-49397ab41c60","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:42:59.542Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"bffdb127-56c1-470d-ad22-49397ab41c60","podName":"long-running-5s9vc-small-group-worker-pldls","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:41:34Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-05T09:42:59.542Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"bffdb127-56c1-470d-ad22-49397ab41c60","podName":"long-running-5s9vc-small-group-worker-pldls","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:41:34Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-05T09:42:59.543Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"bffdb127-56c1-470d-ad22-49397ab41c60","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:42:59.543Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"bffdb127-56c1-470d-ad22-49397ab41c60","seconds":300}
{"level":"info","ts":"2026-06-05T09:43:00.975Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"84f7f2dc-a627-4b7a-af60-726fe7872b74","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:00.975Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"84f7f2dc-a627-4b7a-af60-726fe7872b74","name":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:43:00.975Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"84f7f2dc-a627-4b7a-af60-726fe7872b74","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:00.975Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"84f7f2dc-a627-4b7a-af60-726fe7872b74"}
{"level":"info","ts":"2026-06-05T09:43:00.975Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"84f7f2dc-a627-4b7a-af60-726fe7872b74","RayCluster":"long-running-5s9vc","State":""}
{"level":"info","ts":"2026-06-05T09:43:02.143Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"54c260ea-4992-4437-b003-5e8de8ac8dd0","rayCluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:43:02.143Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"54c260ea-4992-4437-b003-5e8de8ac8dd0","pod":"long-running-5s9vc-head-r7bq6","ip":"10.132.0.20"}
{"level":"info","ts":"2026-06-05T09:43:02.143Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"54c260ea-4992-4437-b003-5e8de8ac8dd0","pod":"long-running-5s9vc-small-group-worker-pldls","ip":"10.133.0.39"}
{"level":"info","ts":"2026-06-05T09:43:02.143Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"54c260ea-4992-4437-b003-5e8de8ac8dd0","rayCluster":"long-running-5s9vc","podIPs":["10.132.0.20","10.133.0.39"]}
{"level":"info","ts":"2026-06-05T09:43:02.215Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"14ff8502-97cf-4163-9a1e-d96250e74426"}
{"level":"info","ts":"2026-06-05T09:43:02.215Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"14ff8502-97cf-4163-9a1e-d96250e74426"}
{"level":"info","ts":"2026-06-05T09:43:02.215Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"14ff8502-97cf-4163-9a1e-d96250e74426","1 head service found":"long-running-5s9vc-head-svc"}
{"level":"info","ts":"2026-06-05T09:43:02.216Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"14ff8502-97cf-4163-9a1e-d96250e74426","Found 1 head Pod":"long-running-5s9vc-head-r7bq6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:43:02.216Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"14ff8502-97cf-4163-9a1e-d96250e74426","head Pod":"long-running-5s9vc-head-r7bq6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-5s9vc-head-r7bq6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:02.216Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"14ff8502-97cf-4163-9a1e-d96250e74426","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:43:02.216Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"14ff8502-97cf-4163-9a1e-d96250e74426","worker Pod":"long-running-5s9vc-small-group-worker-pldls","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-5s9vc-small-group-worker-pldls. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:02.216Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"14ff8502-97cf-4163-9a1e-d96250e74426","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:02.216Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"14ff8502-97cf-4163-9a1e-d96250e74426","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:43:02.216Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"14ff8502-97cf-4163-9a1e-d96250e74426","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:02.216Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"14ff8502-97cf-4163-9a1e-d96250e74426","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-06-05T09:43:02.216Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"14ff8502-97cf-4163-9a1e-d96250e74426","name":"long-running-5s9vc","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:42:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.20","serviceIP":"10.132.0.20","podName":"long-running-5s9vc-head-r7bq6","serviceName":"long-running-5s9vc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:42:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:41:32Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:43:02Z","stateTransitionTimes":{"ready":"2026-06-05T09:43:02Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.20","serviceIP":"10.132.0.20","podName":"long-running-5s9vc-head-r7bq6","serviceName":"long-running-5s9vc-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:42:57Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:43:02Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:41:31Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:41:32Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:43:02.230Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"656e10e0-2c4e-42f6-9cf0-645f60450c27","cluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:43:02.230Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"fdedc3a1-f69c-42fc-be70-d0d9912b862b","rayCluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:43:02.230Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"fdedc3a1-f69c-42fc-be70-d0d9912b862b","pod":"long-running-5s9vc-small-group-worker-pldls","ip":"10.133.0.39"}
{"level":"info","ts":"2026-06-05T09:43:02.230Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"fdedc3a1-f69c-42fc-be70-d0d9912b862b","pod":"long-running-5s9vc-head-r7bq6","ip":"10.132.0.20"}
{"level":"info","ts":"2026-06-05T09:43:02.230Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"ac947c3f-1098-46fe-af7d-fae02aac0431","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:02.230Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"ac947c3f-1098-46fe-af7d-fae02aac0431","name":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:43:02.230Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"ac947c3f-1098-46fe-af7d-fae02aac0431","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:02.230Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"fdedc3a1-f69c-42fc-be70-d0d9912b862b","rayCluster":"long-running-5s9vc","podIPs":["10.133.0.39","10.132.0.20"]}
{"level":"info","ts":"2026-06-05T09:43:02.230Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"ac947c3f-1098-46fe-af7d-fae02aac0431"}
{"level":"info","ts":"2026-06-05T09:43:02.230Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"ac947c3f-1098-46fe-af7d-fae02aac0431","head service name":"long-running-5s9vc-head-svc","namespace":"test-ns-t5s4r"}
{"level":"info","ts":"2026-06-05T09:43:02.230Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"ac947c3f-1098-46fe-af7d-fae02aac0431","head service URL":"long-running-5s9vc-head-svc.test-ns-t5s4r.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-05T09:43:02.230Z","logger":"controllers.RayJob","msg":"user-provided submitter template is used; the first container is assumed to be the submitter","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"ac947c3f-1098-46fe-af7d-fae02aac0431"}
{"level":"info","ts":"2026-06-05T09:43:02.230Z","logger":"controllers.RayJob","msg":"No command is specified in the user-provided template. Default command is used","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"ac947c3f-1098-46fe-af7d-fae02aac0431","command":["if","!","ray","job","status","--address","http://long-running-5s9vc-head-svc.test-ns-t5s4r.svc.cluster.local:8265","long-running-r2shw",">/dev/null","2>&1",";","then","ray","job","submit","--address","http://long-running-5s9vc-head-svc.test-ns-t5s4r.svc.cluster.local:8265","--no-wait","--submission-id","long-running-r2shw","--","python /home/ray/jobs/long_running.py",";","fi",";","ray","job","logs","--address","http://long-running-5s9vc-head-svc.test-ns-t5s4r.svc.cluster.local:8265","--follow","long-running-r2shw"]}
{"level":"info","ts":"2026-06-05T09:43:02.240Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"ac947c3f-1098-46fe-af7d-fae02aac0431","Kubernetes Job":"long-running"}
{"level":"info","ts":"2026-06-05T09:43:02.240Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"ac947c3f-1098-46fe-af7d-fae02aac0431","SubmissionMode":"K8sJobMode","RayCluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:43:02.240Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"ac947c3f-1098-46fe-af7d-fae02aac0431","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r2shw","rayClusterName":"long-running-5s9vc","jobDeploymentStatus":"Initializing","startTime":"2026-06-05T09:41:31Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r2shw","rayClusterName":"long-running-5s9vc","dashboardURL":"long-running-5s9vc-head-svc.test-ns-t5s4r.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-05T09:41:31Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:43:02.240Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"ac947c3f-1098-46fe-af7d-fae02aac0431","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-05T09:43:02.241Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"656e10e0-2c4e-42f6-9cf0-645f60450c27","name":"long-running-5s9vc-head"}
{"level":"info","ts":"2026-06-05T09:43:02.254Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"656e10e0-2c4e-42f6-9cf0-645f60450c27","name":"long-running-5s9vc-workers"}
{"level":"info","ts":"2026-06-05T09:43:02.254Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"656e10e0-2c4e-42f6-9cf0-645f60450c27","cluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:43:02.305Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"1b90ea81-486b-4efc-8275-f37e677cf87f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:02.305Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"1b90ea81-486b-4efc-8275-f37e677cf87f","name":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:43:02.305Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"1b90ea81-486b-4efc-8275-f37e677cf87f","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:02.306Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"1b90ea81-486b-4efc-8275-f37e677cf87f"}
{"level":"info","ts":"2026-06-05T09:43:02.306Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"1b90ea81-486b-4efc-8275-f37e677cf87f","head service name":"long-running-5s9vc-head-svc","namespace":"test-ns-t5s4r"}
{"level":"info","ts":"2026-06-05T09:43:02.306Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"1b90ea81-486b-4efc-8275-f37e677cf87f","head service URL":"long-running-5s9vc-head-svc.test-ns-t5s4r.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-05T09:43:02.306Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job for RayJob already exists","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"1b90ea81-486b-4efc-8275-f37e677cf87f","Kubernetes Job":"long-running"}
{"level":"info","ts":"2026-06-05T09:43:02.306Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"1b90ea81-486b-4efc-8275-f37e677cf87f","SubmissionMode":"K8sJobMode","RayCluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:43:02.306Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"1b90ea81-486b-4efc-8275-f37e677cf87f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r2shw","rayClusterName":"long-running-5s9vc","jobDeploymentStatus":"Initializing","startTime":"2026-06-05T09:41:31Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r2shw","rayClusterName":"long-running-5s9vc","dashboardURL":"long-running-5s9vc-head-svc.test-ns-t5s4r.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-05T09:41:31Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:43:02.306Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"1b90ea81-486b-4efc-8275-f37e677cf87f","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-05T09:43:02.314Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"1b90ea81-486b-4efc-8275-f37e677cf87f","error":"Operation cannot be fulfilled on rayjobs.ray.io \"long-running\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-06-05T09:43:02.314Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"1b90ea81-486b-4efc-8275-f37e677cf87f"}
{"level":"error","ts":"2026-06-05T09:43:02.314Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"1b90ea81-486b-4efc-8275-f37e677cf87f","error":"Operation cannot be fulfilled on rayjobs.ray.io \"long-running\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:02.314Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"446397f6-44c7-446e-951b-f456f00c1923","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:02.314Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"446397f6-44c7-446e-951b-f456f00c1923","name":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:43:02.314Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"446397f6-44c7-446e-951b-f456f00c1923","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:02.314Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"446397f6-44c7-446e-951b-f456f00c1923"}
{"level":"error","ts":"2026-06-05T09:43:02.322Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"446397f6-44c7-446e-951b-f456f00c1923","JobId":"long-running-r2shw","error":"Job long-running-r2shw does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:02.322Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"446397f6-44c7-446e-951b-f456f00c1923"}
{"level":"error","ts":"2026-06-05T09:43:02.322Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"446397f6-44c7-446e-951b-f456f00c1923","error":"Job long-running-r2shw does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:02.323Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"f40bcb83-c95d-4917-bcc5-816ec17f0e0e","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:02.323Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"f40bcb83-c95d-4917-bcc5-816ec17f0e0e","name":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:43:02.323Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"f40bcb83-c95d-4917-bcc5-816ec17f0e0e","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:02.323Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"f40bcb83-c95d-4917-bcc5-816ec17f0e0e"}
{"level":"error","ts":"2026-06-05T09:43:02.327Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"f40bcb83-c95d-4917-bcc5-816ec17f0e0e","JobId":"long-running-r2shw","error":"Job long-running-r2shw does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:02.327Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"f40bcb83-c95d-4917-bcc5-816ec17f0e0e"}
{"level":"error","ts":"2026-06-05T09:43:02.327Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"f40bcb83-c95d-4917-bcc5-816ec17f0e0e","error":"Job long-running-r2shw does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:02.332Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"e33b15e9-97b7-4bb2-bd92-d9f6db869733","DeletionTimestamp":"2026-06-05 09:43:02 +0000 UTC"}
{"level":"info","ts":"2026-06-05T09:43:02.332Z","logger":"controllers.RayJob","msg":"Stop a ray job","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"e33b15e9-97b7-4bb2-bd92-d9f6db869733","rayJob":"long-running-r2shw"}
{"level":"error","ts":"2026-06-05T09:43:02.336Z","logger":"controllers.RayJob","msg":"Failed to stop job for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"e33b15e9-97b7-4bb2-bd92-d9f6db869733","error":"invalid character 'J' looking for beginning of value","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:123\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:02.336Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"e33b15e9-97b7-4bb2-bd92-d9f6db869733","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-05T09:43:02.349Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"549ea2fd-d919-4710-9a16-789d12f07e7e","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:02.349Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"549ea2fd-d919-4710-9a16-789d12f07e7e","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-05T09:43:02.361Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"549ea2fd-d919-4710-9a16-789d12f07e7e"}
{"level":"info","ts":"2026-06-05T09:43:02.361Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"549ea2fd-d919-4710-9a16-789d12f07e7e","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-05T09:43:02.361Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"549ea2fd-d919-4710-9a16-789d12f07e7e","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cflqr","rayClusterName":"counter-6t6xr","jobDeploymentStatus":"Initializing","startTime":"2026-06-05T09:43:02Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:43:02.361Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"549ea2fd-d919-4710-9a16-789d12f07e7e","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-05T09:43:02.406Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"20d0fa75-b06d-4734-8fd0-187f02f674fb"}
{"level":"info","ts":"2026-06-05T09:43:02.406Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"f06d9045-c745-4e6e-b9c6-0715220952c8","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:02.406Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"f06d9045-c745-4e6e-b9c6-0715220952c8"}
{"level":"info","ts":"2026-06-05T09:43:02.406Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"f06d9045-c745-4e6e-b9c6-0715220952c8","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-05T09:43:02.406Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"f06d9045-c745-4e6e-b9c6-0715220952c8","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-z7r6f","rayClusterName":"counter-cssz4","jobDeploymentStatus":"Initializing","startTime":"2026-06-05T09:43:02Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:43:02.406Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"f06d9045-c745-4e6e-b9c6-0715220952c8","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-05T09:43:02.406Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"c9b33c48-2631-4e11-934e-a3e6977d7c9c","rayCluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:43:02.406Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"b65679a4-c73a-4a59-80ac-e40656c208a7"}
{"level":"info","ts":"2026-06-05T09:43:02.406Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"1f6d6fe8-4f52-41e7-8054-a09da0ad997a","namespacedName":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:02.406Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"reconcileID":"dde4fb60-3db2-4b19-8800-714814981601"}
{"level":"info","ts":"2026-06-05T09:43:02.406Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"1f6d6fe8-4f52-41e7-8054-a09da0ad997a","cluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:43:02.406Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"1f6d6fe8-4f52-41e7-8054-a09da0ad997a","cluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:43:02.407Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"c9b33c48-2631-4e11-934e-a3e6977d7c9c","pod":"long-running-5s9vc-head-r7bq6","ip":"10.132.0.20"}
{"level":"info","ts":"2026-06-05T09:43:02.407Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"c9b33c48-2631-4e11-934e-a3e6977d7c9c","pod":"long-running-5s9vc-small-group-worker-pldls","ip":"10.133.0.39"}
{"level":"info","ts":"2026-06-05T09:43:02.407Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"1f6d6fe8-4f52-41e7-8054-a09da0ad997a","namespace":"test-ns-t5s4r","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-05T09:43:02.407Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"c9b33c48-2631-4e11-934e-a3e6977d7c9c","rayCluster":"long-running-5s9vc","podIPs":["10.132.0.20","10.133.0.39"]}
{"level":"info","ts":"2026-06-05T09:43:02.413Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"1f6d6fe8-4f52-41e7-8054-a09da0ad997a","name":"kuberay-gateway-access","namespace":"test-ns-t5s4r"}
{"level":"info","ts":"2026-06-05T09:43:02.414Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"f06d9045-c745-4e6e-b9c6-0715220952c8","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-06-05T09:43:02.414Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"f06d9045-c745-4e6e-b9c6-0715220952c8"}
{"level":"error","ts":"2026-06-05T09:43:02.414Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"f06d9045-c745-4e6e-b9c6-0715220952c8","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:02.414Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"c7607164-1718-471d-b82d-116e630bd6e3"}
{"level":"info","ts":"2026-06-05T09:43:02.414Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"6d48e4ad-8e93-4dd4-8604-d4b17a216979","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:02.414Z","logger":"controllers.RayJob","msg":"Try to transition the status to `Suspending`","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"6d48e4ad-8e93-4dd4-8604-d4b17a216979","oldStatus":"Initializing"}
{"level":"info","ts":"2026-06-05T09:43:02.414Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"6d48e4ad-8e93-4dd4-8604-d4b17a216979","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cflqr","rayClusterName":"counter-6t6xr","jobDeploymentStatus":"Initializing","startTime":"2026-06-05T09:43:02Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cflqr","rayClusterName":"counter-6t6xr","jobDeploymentStatus":"Suspending","startTime":"2026-06-05T09:43:02Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:43:02.414Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"6d48e4ad-8e93-4dd4-8604-d4b17a216979","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Suspending"}
{"level":"info","ts":"2026-06-05T09:43:02.420Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"1f6d6fe8-4f52-41e7-8054-a09da0ad997a","name":"test-ns-t5s4r-long-running-5s9vc","namespace":"opendatahub"}
{"level":"info","ts":"2026-06-05T09:43:02.428Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"1f6d6fe8-4f52-41e7-8054-a09da0ad997a","configMap":"kube-rbac-proxy-config-long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:43:02.428Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"007bb309-922a-4139-87bc-41e88b3a59c6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:02.428Z","logger":"controllers.RayJob","msg":"Try to transition the status to `Suspending`","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"007bb309-922a-4139-87bc-41e88b3a59c6","oldStatus":"Initializing"}
{"level":"info","ts":"2026-06-05T09:43:02.428Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"007bb309-922a-4139-87bc-41e88b3a59c6","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cflqr","rayClusterName":"counter-6t6xr","jobDeploymentStatus":"Initializing","startTime":"2026-06-05T09:43:02Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cflqr","rayClusterName":"counter-6t6xr","jobDeploymentStatus":"Suspending","startTime":"2026-06-05T09:43:02Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:43:02.428Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"007bb309-922a-4139-87bc-41e88b3a59c6","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Suspending"}
{"level":"info","ts":"2026-06-05T09:43:02.433Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"1f6d6fe8-4f52-41e7-8054-a09da0ad997a","serviceAccount":"long-running-5s9vc-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:43:02.435Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"007bb309-922a-4139-87bc-41e88b3a59c6","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-06-05T09:43:02.435Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"007bb309-922a-4139-87bc-41e88b3a59c6"}
{"level":"error","ts":"2026-06-05T09:43:02.435Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"007bb309-922a-4139-87bc-41e88b3a59c6","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:02.435Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"1661b06d-6a8b-44a0-acb9-498f58eff37b","JobStatus":"","JobDeploymentStatus":"Suspending","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:02.435Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"1661b06d-6a8b-44a0-acb9-498f58eff37b","RayCluster":{"name":"counter-6t6xr","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:02.435Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"1661b06d-6a8b-44a0-acb9-498f58eff37b","isClusterDeleted":true}
{"level":"info","ts":"2026-06-05T09:43:02.435Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job has been already deleted","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"1661b06d-6a8b-44a0-acb9-498f58eff37b","Kubernetes Job":""}
{"level":"info","ts":"2026-06-05T09:43:02.435Z","logger":"controllers.RayJob","msg":"deleteSubmitterJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"1661b06d-6a8b-44a0-acb9-498f58eff37b","isJobDeleted":true}
{"level":"info","ts":"2026-06-05T09:43:02.435Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"1661b06d-6a8b-44a0-acb9-498f58eff37b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cflqr","rayClusterName":"counter-6t6xr","jobDeploymentStatus":"Suspending","startTime":"2026-06-05T09:43:02Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobDeploymentStatus":"Suspended","startTime":"2026-06-05T09:43:02Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:43:02.435Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"1661b06d-6a8b-44a0-acb9-498f58eff37b","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Suspending","new JobDeploymentStatus":"Suspended"}
{"level":"info","ts":"2026-06-05T09:43:02.506Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"24adc006-4e97-4e3f-bb3e-6b47666f4fb4","JobStatus":"","JobDeploymentStatus":"Suspending","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:02.506Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"24adc006-4e97-4e3f-bb3e-6b47666f4fb4","RayCluster":{"name":"counter-6t6xr","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:02.506Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"24adc006-4e97-4e3f-bb3e-6b47666f4fb4","isClusterDeleted":true}
{"level":"info","ts":"2026-06-05T09:43:02.506Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job has been already deleted","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"24adc006-4e97-4e3f-bb3e-6b47666f4fb4","Kubernetes Job":""}
{"level":"info","ts":"2026-06-05T09:43:02.506Z","logger":"controllers.RayJob","msg":"deleteSubmitterJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"24adc006-4e97-4e3f-bb3e-6b47666f4fb4","isJobDeleted":true}
{"level":"info","ts":"2026-06-05T09:43:02.506Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"24adc006-4e97-4e3f-bb3e-6b47666f4fb4","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cflqr","rayClusterName":"counter-6t6xr","jobDeploymentStatus":"Suspending","startTime":"2026-06-05T09:43:02Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobDeploymentStatus":"Suspended","startTime":"2026-06-05T09:43:02Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:43:02.506Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"24adc006-4e97-4e3f-bb3e-6b47666f4fb4","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Suspending","new JobDeploymentStatus":"Suspended"}
{"level":"info","ts":"2026-06-05T09:43:02.512Z","logger":"raycluster-resource","msg":"default","name":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:43:02.512Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}
{"level":"info","ts":"2026-06-05T09:43:02.514Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"24adc006-4e97-4e3f-bb3e-6b47666f4fb4","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-06-05T09:43:02.514Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"24adc006-4e97-4e3f-bb3e-6b47666f4fb4"}
{"level":"error","ts":"2026-06-05T09:43:02.514Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"24adc006-4e97-4e3f-bb3e-6b47666f4fb4","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:02.514Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"cc328fce-766a-40c2-8cbd-1fa90874231a","JobStatus":"","JobDeploymentStatus":"Suspended","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:02.519Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"2e854992-f188-47ba-bf93-82da54753c89","JobStatus":"","JobDeploymentStatus":"Suspended","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:02.523Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"1f6d6fe8-4f52-41e7-8054-a09da0ad997a","cluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:43:02.523Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"8bed9333-f932-43a4-8657-fcd11dbb9cdd","namespacedName":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:02.523Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"8bed9333-f932-43a4-8657-fcd11dbb9cdd","cluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:43:02.523Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"8bed9333-f932-43a4-8657-fcd11dbb9cdd","cluster":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:43:02.523Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"8bed9333-f932-43a4-8657-fcd11dbb9cdd","namespace":"test-ns-t5s4r","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-05T09:43:02.523Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"8bed9333-f932-43a4-8657-fcd11dbb9cdd","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-06-05T09:43:02.523Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"353c4f8b-23d4-4ed2-949c-bd87c95a42d0"}
{"level":"info","ts":"2026-06-05T09:43:02.523Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"cb48ad77-4cef-4656-b279-e7176bdb50de","namespace":"test-ns-t5s4r","clusterName":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:43:02.523Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"b9fc0e70-8e99-4692-beff-e00458dc73d9"}
{"level":"info","ts":"2026-06-05T09:43:02.606Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"89a5c142-69de-481c-a378-43a085c8f7a4"}
{"level":"error","ts":"2026-06-05T09:43:02.612Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"8bed9333-f932-43a4-8657-fcd11dbb9cdd","error":"rayclusters.ray.io \"long-running-5s9vc\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:214\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:155\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:02.612Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"8bed9333-f932-43a4-8657-fcd11dbb9cdd"}
{"level":"error","ts":"2026-06-05T09:43:02.612Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"8bed9333-f932-43a4-8657-fcd11dbb9cdd","error":"rayclusters.ray.io \"long-running-5s9vc\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:02.612Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"a1c0a932-47be-43f9-a2c5-632fd8e6e0ee","namespacedName":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:02.612Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"a1c0a932-47be-43f9-a2c5-632fd8e6e0ee"}
{"level":"info","ts":"2026-06-05T09:43:02.612Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"a1c0a932-47be-43f9-a2c5-632fd8e6e0ee","namespace":"test-ns-t5s4r","authClusters":0}
{"level":"info","ts":"2026-06-05T09:43:02.612Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"a1c0a932-47be-43f9-a2c5-632fd8e6e0ee","namespace":"test-ns-t5s4r"}
{"level":"error","ts":"2026-06-05T09:43:02.613Z","msg":"Failed to delete head certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"cb48ad77-4cef-4656-b279-e7176bdb50de","certificate":"ray-head-cert-long-running-5s9vc","error":"certificates.cert-manager.io \"ray-head-cert-long-running-5s9vc\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:261\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:02.618Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"3137cf4d-51b0-423d-a6fe-506bef3aae62","namespacedName":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:02.618Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"3137cf4d-51b0-423d-a6fe-506bef3aae62"}
{"level":"info","ts":"2026-06-05T09:43:02.618Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"3137cf4d-51b0-423d-a6fe-506bef3aae62","namespace":"test-ns-t5s4r","authClusters":0}
{"level":"info","ts":"2026-06-05T09:43:02.618Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"3137cf4d-51b0-423d-a6fe-506bef3aae62","namespace":"test-ns-t5s4r"}
{"level":"info","ts":"2026-06-05T09:43:02.637Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"cb48ad77-4cef-4656-b279-e7176bdb50de","errorCount":1}
{"level":"info","ts":"2026-06-05T09:43:02.637Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"cb48ad77-4cef-4656-b279-e7176bdb50de"}
{"level":"error","ts":"2026-06-05T09:43:02.637Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"cb48ad77-4cef-4656-b279-e7176bdb50de","error":"certificates.cert-manager.io \"ray-head-cert-long-running-5s9vc\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:02.642Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"8c37348d-6370-4a87-8a88-894a0efab9c9","namespace":"test-ns-t5s4r","clusterName":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:43:02.642Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-5s9vc","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"long-running-5s9vc","reconcileID":"8c37348d-6370-4a87-8a88-894a0efab9c9","namespace":"test-ns-t5s4r","clusterName":"long-running-5s9vc"}
{"level":"info","ts":"2026-06-05T09:43:03.369Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"756de00a-1886-42f5-bac3-e20622224bc9","JobStatus":"","JobDeploymentStatus":"Suspended","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:03.369Z","logger":"controllers.RayJob","msg":"The status is 'Suspended', but the suspend flag is false. Transition the status to 'New'.","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"756de00a-1886-42f5-bac3-e20622224bc9"}
{"level":"info","ts":"2026-06-05T09:43:03.369Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"756de00a-1886-42f5-bac3-e20622224bc9","oldRayJobStatus":{"rayJobInfo":{},"jobDeploymentStatus":"Suspended","startTime":"2026-06-05T09:43:02Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"startTime":"2026-06-05T09:43:02Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:43:03.369Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"756de00a-1886-42f5-bac3-e20622224bc9","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Suspended","new JobDeploymentStatus":""}
{"level":"info","ts":"2026-06-05T09:43:03.380Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"c2bcc321-99c8-48aa-a0ba-9df7524c4404","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:03.380Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"c2bcc321-99c8-48aa-a0ba-9df7524c4404"}
{"level":"info","ts":"2026-06-05T09:43:03.380Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"c2bcc321-99c8-48aa-a0ba-9df7524c4404","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-05T09:43:03.380Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"c2bcc321-99c8-48aa-a0ba-9df7524c4404","oldRayJobStatus":{"rayJobInfo":{},"startTime":"2026-06-05T09:43:02Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dgn8t","rayClusterName":"counter-4qmws","jobDeploymentStatus":"Initializing","startTime":"2026-06-05T09:43:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:43:03.380Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"c2bcc321-99c8-48aa-a0ba-9df7524c4404","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-05T09:43:03.393Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"00681f67-abb4-48e7-a21b-f36c44074af6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:03.393Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"00681f67-abb4-48e7-a21b-f36c44074af6","name":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:03.393Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"00681f67-abb4-48e7-a21b-f36c44074af6","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:03.393Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"00681f67-abb4-48e7-a21b-f36c44074af6","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:03.398Z","logger":"raycluster-resource","msg":"default","name":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:03.398Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-4qmws","namespace":"test-ns-t5s4r"}
{"level":"info","ts":"2026-06-05T09:43:03.408Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"00681f67-abb4-48e7-a21b-f36c44074af6","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:03.408Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"d5d5b45a-4b7e-4f6c-a9d2-9f580ce548a5","namespacedName":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:03.408Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"d5d5b45a-4b7e-4f6c-a9d2-9f580ce548a5","mode":"IntegratedOAuth","cluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:03.408Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"d5d5b45a-4b7e-4f6c-a9d2-9f580ce548a5","cluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:03.408Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"8a5f3b1c-484b-4cda-bc97-e10aefe42c87","rayCluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:03.408Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"8a5f3b1c-484b-4cda-bc97-e10aefe42c87","rayCluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:03.408Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"ee5da2e6-187d-4207-a0f3-f072b6708707","cluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:03.408Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"609867f0-e757-4f85-8364-49b82f47a0da"}
{"level":"info","ts":"2026-06-05T09:43:03.408Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"609867f0-e757-4f85-8364-49b82f47a0da"}
{"level":"info","ts":"2026-06-05T09:43:03.408Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"d5d5b45a-4b7e-4f6c-a9d2-9f580ce548a5","cluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:03.408Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"00681f67-abb4-48e7-a21b-f36c44074af6"}
{"level":"info","ts":"2026-06-05T09:43:03.408Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"00681f67-abb4-48e7-a21b-f36c44074af6","RayCluster":"counter-4qmws","State":""}
{"level":"info","ts":"2026-06-05T09:43:03.408Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"2e184f14-5843-4ed6-a863-9bfff82715aa","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:03.408Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"2e184f14-5843-4ed6-a863-9bfff82715aa","name":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:03.408Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"2e184f14-5843-4ed6-a863-9bfff82715aa","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:03.408Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"2e184f14-5843-4ed6-a863-9bfff82715aa"}
{"level":"info","ts":"2026-06-05T09:43:03.408Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"2e184f14-5843-4ed6-a863-9bfff82715aa","RayCluster":"counter-4qmws","State":""}
{"level":"info","ts":"2026-06-05T09:43:03.415Z","logger":"raycluster-resource","msg":"default","name":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:03.415Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-4qmws","namespace":"test-ns-t5s4r"}
{"level":"info","ts":"2026-06-05T09:43:03.417Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"609867f0-e757-4f85-8364-49b82f47a0da","name":"counter-4qmws-head-svc"}
{"level":"info","ts":"2026-06-05T09:43:03.417Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"609867f0-e757-4f85-8364-49b82f47a0da","error":"head certificate secret ray-head-secret-counter-4qmws not found"}
{"level":"error","ts":"2026-06-05T09:43:03.417Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"609867f0-e757-4f85-8364-49b82f47a0da","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-4qmws not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:03.417Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"609867f0-e757-4f85-8364-49b82f47a0da","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-4qmws not found"}
{"level":"info","ts":"2026-06-05T09:43:03.417Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"609867f0-e757-4f85-8364-49b82f47a0da","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"counter-4qmws","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:03.417Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"609867f0-e757-4f85-8364-49b82f47a0da","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"counter-4qmws","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:03.417Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"609867f0-e757-4f85-8364-49b82f47a0da","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"counter-4qmws","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:03.417Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"609867f0-e757-4f85-8364-49b82f47a0da","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-05T09:43:03.417Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"609867f0-e757-4f85-8364-49b82f47a0da","name":"counter-4qmws","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:43:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-4qmws-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:43:03.418Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"8a5f3b1c-484b-4cda-bc97-e10aefe42c87","rayCluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:03.418Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"8a5f3b1c-484b-4cda-bc97-e10aefe42c87","rayCluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:03.425Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"ee5da2e6-187d-4207-a0f3-f072b6708707","cluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:03.425Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"9d06da4f-39ea-4640-864e-a219867570f1","cluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:03.506Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"609867f0-e757-4f85-8364-49b82f47a0da","name":"counter-4qmws","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-4qmws\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-t5s4r","name":"counter-4qmws"}}
{"level":"info","ts":"2026-06-05T09:43:03.506Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"609867f0-e757-4f85-8364-49b82f47a0da"}
{"level":"info","ts":"2026-06-05T09:43:03.506Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"8a5f3b1c-484b-4cda-bc97-e10aefe42c87"}
{"level":"error","ts":"2026-06-05T09:43:03.506Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"609867f0-e757-4f85-8364-49b82f47a0da","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-4qmws not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:03.506Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"0af02f85-94f4-48d2-868d-6547417ca5ad"}
{"level":"info","ts":"2026-06-05T09:43:03.506Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"0af02f85-94f4-48d2-868d-6547417ca5ad"}
{"level":"info","ts":"2026-06-05T09:43:03.506Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"0af02f85-94f4-48d2-868d-6547417ca5ad","1 head service found":"counter-4qmws-head-svc"}
{"level":"info","ts":"2026-06-05T09:43:03.506Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"0af02f85-94f4-48d2-868d-6547417ca5ad","error":"head certificate secret ray-head-secret-counter-4qmws not found"}
{"level":"error","ts":"2026-06-05T09:43:03.506Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"0af02f85-94f4-48d2-868d-6547417ca5ad","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-4qmws not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:03.506Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"0af02f85-94f4-48d2-868d-6547417ca5ad","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-4qmws not found"}
{"level":"info","ts":"2026-06-05T09:43:03.506Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"0af02f85-94f4-48d2-868d-6547417ca5ad","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"counter-4qmws","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:03.506Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"0af02f85-94f4-48d2-868d-6547417ca5ad","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"counter-4qmws","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:03.506Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"8a5f3b1c-484b-4cda-bc97-e10aefe42c87","rayCluster":"counter-4qmws","secret":"ca-secret"}
{"level":"info","ts":"2026-06-05T09:43:03.506Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"0af02f85-94f4-48d2-868d-6547417ca5ad","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"counter-4qmws","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:03.506Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"8a5f3b1c-484b-4cda-bc97-e10aefe42c87","rayCluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:03.506Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"0af02f85-94f4-48d2-868d-6547417ca5ad","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-05T09:43:03.506Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"0af02f85-94f4-48d2-868d-6547417ca5ad","name":"counter-4qmws","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:43:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-4qmws-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:43:03.506Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"fc0c91f7-ca7d-4103-855e-750208cccb16","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:03.506Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"fc0c91f7-ca7d-4103-855e-750208cccb16","name":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:03.506Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"fc0c91f7-ca7d-4103-855e-750208cccb16","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:03.506Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"fc0c91f7-ca7d-4103-855e-750208cccb16"}
{"level":"info","ts":"2026-06-05T09:43:03.507Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"fc0c91f7-ca7d-4103-855e-750208cccb16","RayCluster":"counter-4qmws","State":""}
{"level":"info","ts":"2026-06-05T09:43:03.512Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"9d06da4f-39ea-4640-864e-a219867570f1","name":"counter-4qmws-head"}
{"level":"info","ts":"2026-06-05T09:43:03.512Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"d5d5b45a-4b7e-4f6c-a9d2-9f580ce548a5","name":"counter-4qmws-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:43:03.513Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"0af02f85-94f4-48d2-868d-6547417ca5ad","name":"counter-4qmws","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-4qmws\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-t5s4r","name":"counter-4qmws"}}
{"level":"info","ts":"2026-06-05T09:43:03.513Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"0af02f85-94f4-48d2-868d-6547417ca5ad"}
{"level":"error","ts":"2026-06-05T09:43:03.513Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"0af02f85-94f4-48d2-868d-6547417ca5ad","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-4qmws not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:03.513Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"b9736f57-d7e4-4484-a395-04b2edab9afb"}
{"level":"info","ts":"2026-06-05T09:43:03.513Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"b9736f57-d7e4-4484-a395-04b2edab9afb"}
{"level":"info","ts":"2026-06-05T09:43:03.513Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"b9736f57-d7e4-4484-a395-04b2edab9afb","1 head service found":"counter-4qmws-head-svc"}
{"level":"info","ts":"2026-06-05T09:43:03.513Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"b9736f57-d7e4-4484-a395-04b2edab9afb","error":"head certificate secret ray-head-secret-counter-4qmws not found"}
{"level":"error","ts":"2026-06-05T09:43:03.513Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"b9736f57-d7e4-4484-a395-04b2edab9afb","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-4qmws not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:03.513Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"b9736f57-d7e4-4484-a395-04b2edab9afb","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-4qmws not found"}
{"level":"info","ts":"2026-06-05T09:43:03.513Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"b9736f57-d7e4-4484-a395-04b2edab9afb","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"counter-4qmws","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:03.513Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"b9736f57-d7e4-4484-a395-04b2edab9afb","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"counter-4qmws","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:03.513Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"b9736f57-d7e4-4484-a395-04b2edab9afb","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"counter-4qmws","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:03.513Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"b9736f57-d7e4-4484-a395-04b2edab9afb","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-05T09:43:03.513Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"b9736f57-d7e4-4484-a395-04b2edab9afb","name":"counter-4qmws","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:43:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-4qmws-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:43:03.515Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"8a5f3b1c-484b-4cda-bc97-e10aefe42c87","rayCluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:03.519Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"d5d5b45a-4b7e-4f6c-a9d2-9f580ce548a5","name":"kuberay-gateway-access","namespace":"test-ns-t5s4r","operation":"created"}
{"level":"info","ts":"2026-06-05T09:43:03.524Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"b9736f57-d7e4-4484-a395-04b2edab9afb"}
{"level":"error","ts":"2026-06-05T09:43:03.524Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"b9736f57-d7e4-4484-a395-04b2edab9afb","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-4qmws not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:03.524Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"eb55d573-cd5c-4c37-9d8a-cffba4e5e967"}
{"level":"info","ts":"2026-06-05T09:43:03.524Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"eb55d573-cd5c-4c37-9d8a-cffba4e5e967"}
{"level":"info","ts":"2026-06-05T09:43:03.524Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"90959d0f-0868-4ac4-8917-5e8965bd0a7c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:03.524Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"90959d0f-0868-4ac4-8917-5e8965bd0a7c","name":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:03.524Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"eb55d573-cd5c-4c37-9d8a-cffba4e5e967","1 head service found":"counter-4qmws-head-svc"}
{"level":"info","ts":"2026-06-05T09:43:03.524Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"eb55d573-cd5c-4c37-9d8a-cffba4e5e967","error":"head certificate secret ray-head-secret-counter-4qmws not found"}
{"level":"info","ts":"2026-06-05T09:43:03.524Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"90959d0f-0868-4ac4-8917-5e8965bd0a7c","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"error","ts":"2026-06-05T09:43:03.524Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"eb55d573-cd5c-4c37-9d8a-cffba4e5e967","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-4qmws not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:03.524Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"eb55d573-cd5c-4c37-9d8a-cffba4e5e967","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-4qmws not found"}
{"level":"info","ts":"2026-06-05T09:43:03.524Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"eb55d573-cd5c-4c37-9d8a-cffba4e5e967","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"counter-4qmws","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:03.524Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"eb55d573-cd5c-4c37-9d8a-cffba4e5e967","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"counter-4qmws","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:03.524Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"eb55d573-cd5c-4c37-9d8a-cffba4e5e967","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"counter-4qmws","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:03.524Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"90959d0f-0868-4ac4-8917-5e8965bd0a7c"}
{"level":"info","ts":"2026-06-05T09:43:03.524Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"90959d0f-0868-4ac4-8917-5e8965bd0a7c","RayCluster":"counter-4qmws","State":""}
{"level":"info","ts":"2026-06-05T09:43:03.524Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"eb55d573-cd5c-4c37-9d8a-cffba4e5e967"}
{"level":"error","ts":"2026-06-05T09:43:03.524Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"eb55d573-cd5c-4c37-9d8a-cffba4e5e967","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-4qmws not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:03.526Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"9d06da4f-39ea-4640-864e-a219867570f1","name":"counter-4qmws-workers"}
{"level":"info","ts":"2026-06-05T09:43:03.526Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"9d06da4f-39ea-4640-864e-a219867570f1","cluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:03.526Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"e32f7068-8602-4202-b86d-97e3b900d136","cluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:03.526Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"8a5f3b1c-484b-4cda-bc97-e10aefe42c87"}
{"level":"info","ts":"2026-06-05T09:43:03.526Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"8a5f3b1c-484b-4cda-bc97-e10aefe42c87","rayCluster":"counter-4qmws","podIPs":[]}
{"level":"info","ts":"2026-06-05T09:43:03.606Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"1a064d30-c450-4b0b-9578-e9907f4d5fda"}
{"level":"info","ts":"2026-06-05T09:43:03.606Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"1a064d30-c450-4b0b-9578-e9907f4d5fda"}
{"level":"info","ts":"2026-06-05T09:43:03.606Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"1a064d30-c450-4b0b-9578-e9907f4d5fda","1 head service found":"counter-4qmws-head-svc"}
{"level":"info","ts":"2026-06-05T09:43:03.606Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"1a064d30-c450-4b0b-9578-e9907f4d5fda","error":"head certificate secret ray-head-secret-counter-4qmws not found"}
{"level":"info","ts":"2026-06-05T09:43:03.606Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"d5d5b45a-4b7e-4f6c-a9d2-9f580ce548a5","name":"test-ns-t5s4r-counter-4qmws","operation":"created"}
{"level":"info","ts":"2026-06-05T09:43:03.606Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"8a5f3b1c-484b-4cda-bc97-e10aefe42c87"}
{"level":"error","ts":"2026-06-05T09:43:03.606Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"1a064d30-c450-4b0b-9578-e9907f4d5fda","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-4qmws not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:03.606Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"1a064d30-c450-4b0b-9578-e9907f4d5fda","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-4qmws not found"}
{"level":"info","ts":"2026-06-05T09:43:03.606Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"1a064d30-c450-4b0b-9578-e9907f4d5fda","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"counter-4qmws","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:03.606Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"8a5f3b1c-484b-4cda-bc97-e10aefe42c87","rayCluster":"counter-4qmws","podIPs":[]}
{"level":"info","ts":"2026-06-05T09:43:03.606Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"1a064d30-c450-4b0b-9578-e9907f4d5fda","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"counter-4qmws","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:03.606Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"8a5f3b1c-484b-4cda-bc97-e10aefe42c87"}
{"level":"info","ts":"2026-06-05T09:43:03.606Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"1a064d30-c450-4b0b-9578-e9907f4d5fda","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"counter-4qmws","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:03.606Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"1a064d30-c450-4b0b-9578-e9907f4d5fda"}
{"level":"info","ts":"2026-06-05T09:43:03.606Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"cc5d3b0a-88cb-416c-9401-3262c545f159","rayCluster":"counter-4qmws"}
{"level":"error","ts":"2026-06-05T09:43:03.606Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"1a064d30-c450-4b0b-9578-e9907f4d5fda","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-4qmws not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:03.606Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"cc5d3b0a-88cb-416c-9401-3262c545f159"}
{"level":"info","ts":"2026-06-05T09:43:03.612Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"e32f7068-8602-4202-b86d-97e3b900d136","name":"counter-4qmws-head"}
{"level":"info","ts":"2026-06-05T09:43:03.626Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"e32f7068-8602-4202-b86d-97e3b900d136","name":"counter-4qmws-workers"}
{"level":"info","ts":"2026-06-05T09:43:03.626Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"e32f7068-8602-4202-b86d-97e3b900d136","cluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:03.687Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"3ae2732c-721b-4ae1-8ab3-ea75dd25b6ea"}
{"level":"info","ts":"2026-06-05T09:43:03.687Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"3ae2732c-721b-4ae1-8ab3-ea75dd25b6ea"}
{"level":"info","ts":"2026-06-05T09:43:03.612Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"d5d5b45a-4b7e-4f6c-a9d2-9f580ce548a5","name":"kube-rbac-proxy-config-counter-4qmws","operation":"created"}
{"level":"info","ts":"2026-06-05T09:43:03.705Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"3ae2732c-721b-4ae1-8ab3-ea75dd25b6ea","1 head service found":"counter-4qmws-head-svc"}
{"level":"info","ts":"2026-06-05T09:43:03.705Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"3ae2732c-721b-4ae1-8ab3-ea75dd25b6ea","error":"head certificate secret ray-head-secret-counter-4qmws not found"}
{"level":"error","ts":"2026-06-05T09:43:03.705Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"3ae2732c-721b-4ae1-8ab3-ea75dd25b6ea","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-4qmws not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:03.705Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"3ae2732c-721b-4ae1-8ab3-ea75dd25b6ea","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-4qmws not found"}
{"level":"info","ts":"2026-06-05T09:43:03.706Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"3ae2732c-721b-4ae1-8ab3-ea75dd25b6ea","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"counter-4qmws","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:03.706Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"3ae2732c-721b-4ae1-8ab3-ea75dd25b6ea","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"counter-4qmws","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:03.706Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"3ae2732c-721b-4ae1-8ab3-ea75dd25b6ea","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"counter-4qmws","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:03.706Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"3ae2732c-721b-4ae1-8ab3-ea75dd25b6ea"}
{"level":"error","ts":"2026-06-05T09:43:03.706Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"3ae2732c-721b-4ae1-8ab3-ea75dd25b6ea","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-4qmws not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-05T09:43:03.712Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"d5d5b45a-4b7e-4f6c-a9d2-9f580ce548a5","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-4qmws\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:03.712Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"d5d5b45a-4b7e-4f6c-a9d2-9f580ce548a5"}
{"level":"error","ts":"2026-06-05T09:43:03.712Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"d5d5b45a-4b7e-4f6c-a9d2-9f580ce548a5","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-4qmws\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:03.713Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"754ea62b-4168-4796-a29b-63208ccd461d","namespacedName":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:03.713Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"754ea62b-4168-4796-a29b-63208ccd461d","mode":"IntegratedOAuth","cluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:03.713Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"754ea62b-4168-4796-a29b-63208ccd461d","cluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:03.726Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"754ea62b-4168-4796-a29b-63208ccd461d","cluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:03.726Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"18fa6708-6c06-4f13-8585-af6826c036c1","namespacedName":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:03.726Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"18fa6708-6c06-4f13-8585-af6826c036c1","mode":"IntegratedOAuth","cluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:03.726Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"18fa6708-6c06-4f13-8585-af6826c036c1","cluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:03.726Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"067cba35-95e8-41d4-9b53-e3e32bddd0cd","rayCluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:03.726Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"74399af8-1d70-4862-a027-5c3d8bf40d0e","cluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:03.726Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"f6fad48d-ebd4-439e-8dcd-2214f643f30c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:03.726Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"f6fad48d-ebd4-439e-8dcd-2214f643f30c","name":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:03.726Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"f6fad48d-ebd4-439e-8dcd-2214f643f30c","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:03.726Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"067cba35-95e8-41d4-9b53-e3e32bddd0cd"}
{"level":"info","ts":"2026-06-05T09:43:03.726Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"f6fad48d-ebd4-439e-8dcd-2214f643f30c"}
{"level":"info","ts":"2026-06-05T09:43:03.726Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"f6fad48d-ebd4-439e-8dcd-2214f643f30c","RayCluster":"counter-4qmws","State":""}
{"level":"error","ts":"2026-06-05T09:43:03.733Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"18fa6708-6c06-4f13-8585-af6826c036c1","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-4qmws\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:03.733Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"18fa6708-6c06-4f13-8585-af6826c036c1"}
{"level":"error","ts":"2026-06-05T09:43:03.733Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"18fa6708-6c06-4f13-8585-af6826c036c1","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-4qmws\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:03.738Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"8d7205c2-33ee-4ffd-abc9-b3f2ac02b7aa","namespacedName":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:03.738Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"8d7205c2-33ee-4ffd-abc9-b3f2ac02b7aa","mode":"IntegratedOAuth","cluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:03.738Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"8d7205c2-33ee-4ffd-abc9-b3f2ac02b7aa","cluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:03.741Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"74399af8-1d70-4862-a027-5c3d8bf40d0e","name":"counter-4qmws-head"}
{"level":"info","ts":"2026-06-05T09:43:03.750Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"8d7205c2-33ee-4ffd-abc9-b3f2ac02b7aa","cluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:03.753Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"74399af8-1d70-4862-a027-5c3d8bf40d0e","name":"counter-4qmws-workers"}
{"level":"info","ts":"2026-06-05T09:43:03.753Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"74399af8-1d70-4862-a027-5c3d8bf40d0e","cluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:03.866Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"426cd576-8788-4628-a1a2-7c5bfb16a466"}
{"level":"info","ts":"2026-06-05T09:43:03.866Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"426cd576-8788-4628-a1a2-7c5bfb16a466"}
{"level":"info","ts":"2026-06-05T09:43:03.866Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"426cd576-8788-4628-a1a2-7c5bfb16a466","1 head service found":"counter-4qmws-head-svc"}
{"level":"info","ts":"2026-06-05T09:43:03.866Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"426cd576-8788-4628-a1a2-7c5bfb16a466","error":"head certificate secret ray-head-secret-counter-4qmws not found"}
{"level":"error","ts":"2026-06-05T09:43:03.866Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"426cd576-8788-4628-a1a2-7c5bfb16a466","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-4qmws not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:03.866Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"426cd576-8788-4628-a1a2-7c5bfb16a466","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-4qmws not found"}
{"level":"info","ts":"2026-06-05T09:43:03.866Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"426cd576-8788-4628-a1a2-7c5bfb16a466","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"counter-4qmws","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:03.866Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"426cd576-8788-4628-a1a2-7c5bfb16a466","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"counter-4qmws","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:03.866Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"426cd576-8788-4628-a1a2-7c5bfb16a466","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"counter-4qmws","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:03.866Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"426cd576-8788-4628-a1a2-7c5bfb16a466"}
{"level":"error","ts":"2026-06-05T09:43:03.866Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"426cd576-8788-4628-a1a2-7c5bfb16a466","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-4qmws not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:04.187Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"01817c81-124e-4090-a055-3d1944c1dad6"}
{"level":"info","ts":"2026-06-05T09:43:04.187Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"01817c81-124e-4090-a055-3d1944c1dad6"}
{"level":"info","ts":"2026-06-05T09:43:04.187Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"01817c81-124e-4090-a055-3d1944c1dad6","1 head service found":"counter-4qmws-head-svc"}
{"level":"info","ts":"2026-06-05T09:43:04.187Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"01817c81-124e-4090-a055-3d1944c1dad6","error":"head certificate secret ray-head-secret-counter-4qmws not found"}
{"level":"error","ts":"2026-06-05T09:43:04.187Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"01817c81-124e-4090-a055-3d1944c1dad6","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-4qmws not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:04.187Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"01817c81-124e-4090-a055-3d1944c1dad6","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-4qmws not found"}
{"level":"info","ts":"2026-06-05T09:43:04.187Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"01817c81-124e-4090-a055-3d1944c1dad6","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"counter-4qmws","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:04.187Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"01817c81-124e-4090-a055-3d1944c1dad6","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"counter-4qmws","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:04.187Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"01817c81-124e-4090-a055-3d1944c1dad6","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"counter-4qmws","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:04.187Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"01817c81-124e-4090-a055-3d1944c1dad6"}
{"level":"error","ts":"2026-06-05T09:43:04.187Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"01817c81-124e-4090-a055-3d1944c1dad6","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-4qmws not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:04.829Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"0564eb4e-a947-4453-ad59-59371527bd28"}
{"level":"info","ts":"2026-06-05T09:43:04.829Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"0564eb4e-a947-4453-ad59-59371527bd28"}
{"level":"info","ts":"2026-06-05T09:43:04.829Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"0564eb4e-a947-4453-ad59-59371527bd28","1 head service found":"counter-4qmws-head-svc"}
{"level":"info","ts":"2026-06-05T09:43:04.829Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"0564eb4e-a947-4453-ad59-59371527bd28","error":"head certificate secret ray-head-secret-counter-4qmws not found"}
{"level":"error","ts":"2026-06-05T09:43:04.829Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"0564eb4e-a947-4453-ad59-59371527bd28","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-4qmws not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:04.829Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"0564eb4e-a947-4453-ad59-59371527bd28","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-4qmws not found"}
{"level":"info","ts":"2026-06-05T09:43:04.829Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"0564eb4e-a947-4453-ad59-59371527bd28","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"counter-4qmws","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:04.829Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"0564eb4e-a947-4453-ad59-59371527bd28","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"counter-4qmws","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:04.829Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"0564eb4e-a947-4453-ad59-59371527bd28","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"counter-4qmws","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:04.829Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"0564eb4e-a947-4453-ad59-59371527bd28"}
{"level":"error","ts":"2026-06-05T09:43:04.829Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"0564eb4e-a947-4453-ad59-59371527bd28","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-4qmws not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:05.350Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"long-running","namespace":"test-ns-t5s4r"},"reconcileID":"bfc635ab-b6dd-442f-9875-05b843f16e98"}
{"level":"info","ts":"2026-06-05T09:43:05.520Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"b3a30b42-272a-41de-ad08-4f668cd75f04","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:05.521Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"b3a30b42-272a-41de-ad08-4f668cd75f04","name":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:05.521Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"b3a30b42-272a-41de-ad08-4f668cd75f04","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:05.521Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"b3a30b42-272a-41de-ad08-4f668cd75f04"}
{"level":"info","ts":"2026-06-05T09:43:05.521Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"b3a30b42-272a-41de-ad08-4f668cd75f04","RayCluster":"counter-4qmws","State":""}
{"level":"info","ts":"2026-06-05T09:43:06.110Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"9e2aa84e-e801-4133-bd79-8822f35560e9"}
{"level":"info","ts":"2026-06-05T09:43:06.110Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"9e2aa84e-e801-4133-bd79-8822f35560e9"}
{"level":"info","ts":"2026-06-05T09:43:06.110Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"9e2aa84e-e801-4133-bd79-8822f35560e9","1 head service found":"counter-4qmws-head-svc"}
{"level":"info","ts":"2026-06-05T09:43:06.110Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"9e2aa84e-e801-4133-bd79-8822f35560e9","error":"head certificate secret ray-head-secret-counter-4qmws not found"}
{"level":"error","ts":"2026-06-05T09:43:06.110Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"9e2aa84e-e801-4133-bd79-8822f35560e9","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-4qmws not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:06.110Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"9e2aa84e-e801-4133-bd79-8822f35560e9","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-4qmws not found"}
{"level":"info","ts":"2026-06-05T09:43:06.110Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"9e2aa84e-e801-4133-bd79-8822f35560e9","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"counter-4qmws","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:06.110Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"9e2aa84e-e801-4133-bd79-8822f35560e9","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"counter-4qmws","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:06.110Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"9e2aa84e-e801-4133-bd79-8822f35560e9","filter labels":["test-ns-t5s4r",{"ray.io/cluster":"counter-4qmws","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:06.110Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"9e2aa84e-e801-4133-bd79-8822f35560e9"}
{"level":"error","ts":"2026-06-05T09:43:06.110Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"9e2aa84e-e801-4133-bd79-8822f35560e9","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-4qmws not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:08.522Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"52f922b2-2643-49b0-ac49-72a0806f57a6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:08.522Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"52f922b2-2643-49b0-ac49-72a0806f57a6","name":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:08.522Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"52f922b2-2643-49b0-ac49-72a0806f57a6","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:08.522Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"52f922b2-2643-49b0-ac49-72a0806f57a6"}
{"level":"info","ts":"2026-06-05T09:43:08.522Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"52f922b2-2643-49b0-ac49-72a0806f57a6","RayCluster":"counter-4qmws","State":""}
{"level":"info","ts":"2026-06-05T09:43:08.670Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"96d5c981-ef04-4663-a5fd-dc339a11bdb8"}
{"level":"info","ts":"2026-06-05T09:43:08.670Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"96d5c981-ef04-4663-a5fd-dc339a11bdb8"}
{"level":"info","ts":"2026-06-05T09:43:08.670Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"96d5c981-ef04-4663-a5fd-dc339a11bdb8","1 head service found":"counter-4qmws-head-svc"}
{"level":"info","ts":"2026-06-05T09:43:08.670Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"96d5c981-ef04-4663-a5fd-dc339a11bdb8"}
{"level":"info","ts":"2026-06-05T09:43:08.670Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"96d5c981-ef04-4663-a5fd-dc339a11bdb8","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:43:08.670Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"96d5c981-ef04-4663-a5fd-dc339a11bdb8"}
{"level":"info","ts":"2026-06-05T09:43:08.670Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"96d5c981-ef04-4663-a5fd-dc339a11bdb8","mode":"IntegratedOAuth","cluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:08.670Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"96d5c981-ef04-4663-a5fd-dc339a11bdb8","cluster":"counter-4qmws","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:43:08.670Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"96d5c981-ef04-4663-a5fd-dc339a11bdb8","cluster":"counter-4qmws","authType":"IntegratedOAuth","serviceAccount":"counter-4qmws-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-05T09:43:08.670Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"96d5c981-ef04-4663-a5fd-dc339a11bdb8","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"counter-4qmws","ray.io/group":"headgroup","ray.io/identifier":"counter-4qmws-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-05T09:43:08.670Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"96d5c981-ef04-4663-a5fd-dc339a11bdb8","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"counter-4qmws-head-svc.test-ns-t5s4r.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-05T09:43:08.671Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"96d5c981-ef04-4663-a5fd-dc339a11bdb8","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=counter-4qmws-head-svc.test-ns-t5s4r.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-05T09:43:08.671Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"96d5c981-ef04-4663-a5fd-dc339a11bdb8","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=counter-4qmws-head-svc.test-ns-t5s4r.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-05T09:43:08.671Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"96d5c981-ef04-4663-a5fd-dc339a11bdb8","enabled":true}
{"level":"info","ts":"2026-06-05T09:43:08.688Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"96d5c981-ef04-4663-a5fd-dc339a11bdb8","name":"counter-4qmws-head-gfw44"}
{"level":"info","ts":"2026-06-05T09:43:08.688Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"96d5c981-ef04-4663-a5fd-dc339a11bdb8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:43:08.688Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"96d5c981-ef04-4663-a5fd-dc339a11bdb8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:08.688Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"96d5c981-ef04-4663-a5fd-dc339a11bdb8","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-05T09:43:08.688Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"96d5c981-ef04-4663-a5fd-dc339a11bdb8","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:08.688Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"96d5c981-ef04-4663-a5fd-dc339a11bdb8","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-05T09:43:08.688Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"96d5c981-ef04-4663-a5fd-dc339a11bdb8"}
{"level":"info","ts":"2026-06-05T09:43:08.688Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"96d5c981-ef04-4663-a5fd-dc339a11bdb8","nodeType":"worker","rayStartParams":{"address":"counter-4qmws-head-svc.test-ns-t5s4r.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-05T09:43:08.688Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"96d5c981-ef04-4663-a5fd-dc339a11bdb8","rayStartCmd":"ray start  --address=counter-4qmws-head-svc.test-ns-t5s4r.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:43:08.688Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"96d5c981-ef04-4663-a5fd-dc339a11bdb8","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=counter-4qmws-head-svc.test-ns-t5s4r.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:43:08.688Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"96d5c981-ef04-4663-a5fd-dc339a11bdb8","enabled":true}
{"level":"info","ts":"2026-06-05T09:43:08.704Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"96d5c981-ef04-4663-a5fd-dc339a11bdb8","name":"counter-4qmws-small-group-worker-n29x6"}
{"level":"info","ts":"2026-06-05T09:43:08.704Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"96d5c981-ef04-4663-a5fd-dc339a11bdb8","podName":"counter-4qmws-head-gfw44","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:43:08.705Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"96d5c981-ef04-4663-a5fd-dc339a11bdb8","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"counter-4qmws-head-svc"},"newHead":{"podName":"counter-4qmws-head-gfw44","serviceName":"counter-4qmws-head-svc"}}
{"level":"info","ts":"2026-06-05T09:43:08.705Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"96d5c981-ef04-4663-a5fd-dc339a11bdb8","name":"counter-4qmws","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:43:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-4qmws-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:43:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-4qmws-head-gfw44","serviceName":"counter-4qmws-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:43:08.718Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"5d040585-260b-49d3-b7a3-94b1cec7fbcb"}
{"level":"info","ts":"2026-06-05T09:43:08.718Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"5d040585-260b-49d3-b7a3-94b1cec7fbcb"}
{"level":"info","ts":"2026-06-05T09:43:08.718Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"5d040585-260b-49d3-b7a3-94b1cec7fbcb","1 head service found":"counter-4qmws-head-svc"}
{"level":"info","ts":"2026-06-05T09:43:08.718Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"cc305fcb-a0a3-4353-97b3-b53912f7a888","rayCluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:08.718Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"f13b5c0f-e249-4fe5-a04a-a2ef1ae8d825","cluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:08.718Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"5d040585-260b-49d3-b7a3-94b1cec7fbcb","Found 1 head Pod":"counter-4qmws-head-gfw44","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:43:08.718Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"5d040585-260b-49d3-b7a3-94b1cec7fbcb","head Pod":"counter-4qmws-head-gfw44","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-4qmws-head-gfw44. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:08.718Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"5d040585-260b-49d3-b7a3-94b1cec7fbcb","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:43:08.718Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"5d040585-260b-49d3-b7a3-94b1cec7fbcb","worker Pod":"counter-4qmws-small-group-worker-n29x6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-4qmws-small-group-worker-n29x6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:08.718Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"5d040585-260b-49d3-b7a3-94b1cec7fbcb","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:08.718Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"5d040585-260b-49d3-b7a3-94b1cec7fbcb","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:43:08.718Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"5d040585-260b-49d3-b7a3-94b1cec7fbcb","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:08.718Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"cc305fcb-a0a3-4353-97b3-b53912f7a888","rayCluster":"counter-4qmws","podIPs":[]}
{"level":"info","ts":"2026-06-05T09:43:08.718Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"5d040585-260b-49d3-b7a3-94b1cec7fbcb","podName":"counter-4qmws-head-gfw44","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:43:08.718Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"7dfbc924-ad81-4efb-8b24-78d33adbe086","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:08.718Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"5d040585-260b-49d3-b7a3-94b1cec7fbcb","podName":"counter-4qmws-head-gfw44","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:43:08.718Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"7dfbc924-ad81-4efb-8b24-78d33adbe086","name":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:08.718Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"7dfbc924-ad81-4efb-8b24-78d33adbe086","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:08.718Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"5d040585-260b-49d3-b7a3-94b1cec7fbcb","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:43:08.718Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"5d040585-260b-49d3-b7a3-94b1cec7fbcb","seconds":300}
{"level":"info","ts":"2026-06-05T09:43:08.718Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"7dfbc924-ad81-4efb-8b24-78d33adbe086"}
{"level":"info","ts":"2026-06-05T09:43:08.718Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"7dfbc924-ad81-4efb-8b24-78d33adbe086","RayCluster":"counter-4qmws","State":""}
{"level":"info","ts":"2026-06-05T09:43:08.729Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"f9127bbb-ec51-4d51-b9ce-8540cf36886b"}
{"level":"info","ts":"2026-06-05T09:43:08.729Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"f9127bbb-ec51-4d51-b9ce-8540cf36886b"}
{"level":"info","ts":"2026-06-05T09:43:08.729Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"f9127bbb-ec51-4d51-b9ce-8540cf36886b","1 head service found":"counter-4qmws-head-svc"}
{"level":"info","ts":"2026-06-05T09:43:08.729Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"f9127bbb-ec51-4d51-b9ce-8540cf36886b","Found 1 head Pod":"counter-4qmws-head-gfw44","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:43:08.729Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"f9127bbb-ec51-4d51-b9ce-8540cf36886b","head Pod":"counter-4qmws-head-gfw44","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-4qmws-head-gfw44. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:08.729Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"f9127bbb-ec51-4d51-b9ce-8540cf36886b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:43:08.729Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"f9127bbb-ec51-4d51-b9ce-8540cf36886b","worker Pod":"counter-4qmws-small-group-worker-n29x6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-4qmws-small-group-worker-n29x6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:08.729Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"f9127bbb-ec51-4d51-b9ce-8540cf36886b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:08.729Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"f9127bbb-ec51-4d51-b9ce-8540cf36886b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:43:08.729Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"f9127bbb-ec51-4d51-b9ce-8540cf36886b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:08.729Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"f9127bbb-ec51-4d51-b9ce-8540cf36886b","podName":"counter-4qmws-head-gfw44","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:43:08.729Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"f9127bbb-ec51-4d51-b9ce-8540cf36886b","podName":"counter-4qmws-head-gfw44","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:43:08.729Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"f9127bbb-ec51-4d51-b9ce-8540cf36886b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:43:08.729Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"f9127bbb-ec51-4d51-b9ce-8540cf36886b","seconds":300}
{"level":"info","ts":"2026-06-05T09:43:08.732Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"5ea83a85-8328-4c36-b0a4-7f14befafbb8"}
{"level":"info","ts":"2026-06-05T09:43:08.732Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"5ea83a85-8328-4c36-b0a4-7f14befafbb8"}
{"level":"info","ts":"2026-06-05T09:43:08.733Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"5ea83a85-8328-4c36-b0a4-7f14befafbb8","1 head service found":"counter-4qmws-head-svc"}
{"level":"info","ts":"2026-06-05T09:43:08.733Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"5ea83a85-8328-4c36-b0a4-7f14befafbb8","Found 1 head Pod":"counter-4qmws-head-gfw44","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:43:08.733Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"5ea83a85-8328-4c36-b0a4-7f14befafbb8","head Pod":"counter-4qmws-head-gfw44","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-4qmws-head-gfw44. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:08.733Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"5ea83a85-8328-4c36-b0a4-7f14befafbb8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:43:08.733Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"5ea83a85-8328-4c36-b0a4-7f14befafbb8","worker Pod":"counter-4qmws-small-group-worker-n29x6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-4qmws-small-group-worker-n29x6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:08.733Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"5ea83a85-8328-4c36-b0a4-7f14befafbb8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:08.733Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"5ea83a85-8328-4c36-b0a4-7f14befafbb8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:43:08.733Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"5ea83a85-8328-4c36-b0a4-7f14befafbb8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:08.733Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"5ea83a85-8328-4c36-b0a4-7f14befafbb8","podName":"counter-4qmws-head-gfw44","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:43:08.733Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"5ea83a85-8328-4c36-b0a4-7f14befafbb8","podName":"counter-4qmws-head-gfw44","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:43:08.733Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"5ea83a85-8328-4c36-b0a4-7f14befafbb8","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-05T09:43:08.733Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"5ea83a85-8328-4c36-b0a4-7f14befafbb8","name":"counter-4qmws","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:43:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-4qmws-head-gfw44","serviceName":"counter-4qmws-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:43:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-4qmws-head-gfw44","serviceName":"counter-4qmws-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:43:08.735Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"f13b5c0f-e249-4fe5-a04a-a2ef1ae8d825","name":"counter-4qmws-head"}
{"level":"info","ts":"2026-06-05T09:43:08.745Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"c7881b8a-3d84-4ab6-96ea-7c06a9228c5b","rayCluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:08.745Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"419fea83-e2e7-4732-b3f2-533f44eb6aaa","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:08.745Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"419fea83-e2e7-4732-b3f2-533f44eb6aaa","name":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:08.746Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"419fea83-e2e7-4732-b3f2-533f44eb6aaa","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:08.746Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"419fea83-e2e7-4732-b3f2-533f44eb6aaa"}
{"level":"info","ts":"2026-06-05T09:43:08.746Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"419fea83-e2e7-4732-b3f2-533f44eb6aaa","RayCluster":"counter-4qmws","State":""}
{"level":"info","ts":"2026-06-05T09:43:08.746Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"f13b5c0f-e249-4fe5-a04a-a2ef1ae8d825","name":"counter-4qmws-workers"}
{"level":"info","ts":"2026-06-05T09:43:08.746Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"f13b5c0f-e249-4fe5-a04a-a2ef1ae8d825","cluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:08.746Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"52934172-f147-4acb-8b3e-31823b1619b0","cluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:08.806Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"c7881b8a-3d84-4ab6-96ea-7c06a9228c5b","rayCluster":"counter-4qmws","podIPs":[]}
{"level":"info","ts":"2026-06-05T09:43:08.806Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"f9236f61-aa06-4311-822d-7ddb829366ca"}
{"level":"info","ts":"2026-06-05T09:43:08.806Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"f9236f61-aa06-4311-822d-7ddb829366ca"}
{"level":"info","ts":"2026-06-05T09:43:08.806Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"f9236f61-aa06-4311-822d-7ddb829366ca","1 head service found":"counter-4qmws-head-svc"}
{"level":"info","ts":"2026-06-05T09:43:08.806Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"f9236f61-aa06-4311-822d-7ddb829366ca","Found 1 head Pod":"counter-4qmws-head-gfw44","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:43:08.806Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"f9236f61-aa06-4311-822d-7ddb829366ca","head Pod":"counter-4qmws-head-gfw44","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-4qmws-head-gfw44. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:08.806Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"f9236f61-aa06-4311-822d-7ddb829366ca","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:43:08.806Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"f9236f61-aa06-4311-822d-7ddb829366ca","worker Pod":"counter-4qmws-small-group-worker-n29x6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-4qmws-small-group-worker-n29x6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:08.806Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"f9236f61-aa06-4311-822d-7ddb829366ca","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:08.806Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"f9236f61-aa06-4311-822d-7ddb829366ca","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:43:08.806Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"f9236f61-aa06-4311-822d-7ddb829366ca","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:08.806Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"f9236f61-aa06-4311-822d-7ddb829366ca","podName":"counter-4qmws-head-gfw44","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:43:08.806Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"f9236f61-aa06-4311-822d-7ddb829366ca","podName":"counter-4qmws-head-gfw44","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:43:08.806Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"f9236f61-aa06-4311-822d-7ddb829366ca","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:43:08.806Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"f9236f61-aa06-4311-822d-7ddb829366ca","seconds":300}
{"level":"info","ts":"2026-06-05T09:43:08.819Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"52934172-f147-4acb-8b3e-31823b1619b0","name":"counter-4qmws-head"}
{"level":"info","ts":"2026-06-05T09:43:08.831Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"52934172-f147-4acb-8b3e-31823b1619b0","name":"counter-4qmws-workers"}
{"level":"info","ts":"2026-06-05T09:43:08.831Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"52934172-f147-4acb-8b3e-31823b1619b0","cluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:09.137Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"0b276bba-cae0-4934-a18d-27b47b0a6e71"}
{"level":"info","ts":"2026-06-05T09:43:09.137Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"0b276bba-cae0-4934-a18d-27b47b0a6e71"}
{"level":"info","ts":"2026-06-05T09:43:09.137Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"0b276bba-cae0-4934-a18d-27b47b0a6e71","1 head service found":"counter-4qmws-head-svc"}
{"level":"info","ts":"2026-06-05T09:43:09.137Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"0b276bba-cae0-4934-a18d-27b47b0a6e71","Found 1 head Pod":"counter-4qmws-head-gfw44","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:43:09.137Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"0b276bba-cae0-4934-a18d-27b47b0a6e71","head Pod":"counter-4qmws-head-gfw44","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-4qmws-head-gfw44. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:09.137Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"0b276bba-cae0-4934-a18d-27b47b0a6e71","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:43:09.137Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"0b276bba-cae0-4934-a18d-27b47b0a6e71","worker Pod":"counter-4qmws-small-group-worker-n29x6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-4qmws-small-group-worker-n29x6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:09.137Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"0b276bba-cae0-4934-a18d-27b47b0a6e71","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:09.137Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"0b276bba-cae0-4934-a18d-27b47b0a6e71","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:43:09.137Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"0b276bba-cae0-4934-a18d-27b47b0a6e71","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:09.137Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"0b276bba-cae0-4934-a18d-27b47b0a6e71","podName":"counter-4qmws-head-gfw44","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:43:09.137Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"0b276bba-cae0-4934-a18d-27b47b0a6e71","podName":"counter-4qmws-head-gfw44","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:43:09.137Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"0b276bba-cae0-4934-a18d-27b47b0a6e71","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:43:09.137Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"0b276bba-cae0-4934-a18d-27b47b0a6e71","seconds":300}
{"level":"info","ts":"2026-06-05T09:43:09.174Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"82fbe89f-cc1a-4c4c-aebc-67089b179d6d"}
{"level":"info","ts":"2026-06-05T09:43:09.174Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"82fbe89f-cc1a-4c4c-aebc-67089b179d6d"}
{"level":"info","ts":"2026-06-05T09:43:09.174Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"82fbe89f-cc1a-4c4c-aebc-67089b179d6d","1 head service found":"counter-4qmws-head-svc"}
{"level":"info","ts":"2026-06-05T09:43:09.174Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"82fbe89f-cc1a-4c4c-aebc-67089b179d6d","Found 1 head Pod":"counter-4qmws-head-gfw44","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:43:09.174Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"82fbe89f-cc1a-4c4c-aebc-67089b179d6d","head Pod":"counter-4qmws-head-gfw44","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-4qmws-head-gfw44. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:09.174Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"82fbe89f-cc1a-4c4c-aebc-67089b179d6d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:43:09.174Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"82fbe89f-cc1a-4c4c-aebc-67089b179d6d","worker Pod":"counter-4qmws-small-group-worker-n29x6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-4qmws-small-group-worker-n29x6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:09.174Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"82fbe89f-cc1a-4c4c-aebc-67089b179d6d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:09.174Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"82fbe89f-cc1a-4c4c-aebc-67089b179d6d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:43:09.174Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"82fbe89f-cc1a-4c4c-aebc-67089b179d6d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:09.174Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"82fbe89f-cc1a-4c4c-aebc-67089b179d6d","podName":"counter-4qmws-head-gfw44","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:43:09.175Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"82fbe89f-cc1a-4c4c-aebc-67089b179d6d","podName":"counter-4qmws-head-gfw44","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:43:09.175Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"82fbe89f-cc1a-4c4c-aebc-67089b179d6d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:43:09.175Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"82fbe89f-cc1a-4c4c-aebc-67089b179d6d","seconds":300}
{"level":"info","ts":"2026-06-05T09:43:09.281Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"1cddad0d-8bd3-4d4f-ad2e-1a8c75ab80a6"}
{"level":"info","ts":"2026-06-05T09:43:09.281Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"1cddad0d-8bd3-4d4f-ad2e-1a8c75ab80a6"}
{"level":"info","ts":"2026-06-05T09:43:09.281Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"1cddad0d-8bd3-4d4f-ad2e-1a8c75ab80a6","1 head service found":"counter-4qmws-head-svc"}
{"level":"info","ts":"2026-06-05T09:43:09.281Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"1cddad0d-8bd3-4d4f-ad2e-1a8c75ab80a6","Found 1 head Pod":"counter-4qmws-head-gfw44","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:43:09.281Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"1cddad0d-8bd3-4d4f-ad2e-1a8c75ab80a6","head Pod":"counter-4qmws-head-gfw44","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-4qmws-head-gfw44. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:09.281Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"1cddad0d-8bd3-4d4f-ad2e-1a8c75ab80a6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:43:09.281Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"1cddad0d-8bd3-4d4f-ad2e-1a8c75ab80a6","worker Pod":"counter-4qmws-small-group-worker-n29x6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-4qmws-small-group-worker-n29x6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:09.281Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"1cddad0d-8bd3-4d4f-ad2e-1a8c75ab80a6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:09.281Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"1cddad0d-8bd3-4d4f-ad2e-1a8c75ab80a6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:43:09.281Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"1cddad0d-8bd3-4d4f-ad2e-1a8c75ab80a6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:09.281Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"1cddad0d-8bd3-4d4f-ad2e-1a8c75ab80a6","podName":"counter-4qmws-head-gfw44","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:43:09.281Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"1cddad0d-8bd3-4d4f-ad2e-1a8c75ab80a6","podName":"counter-4qmws-head-gfw44","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:43:09.281Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"1cddad0d-8bd3-4d4f-ad2e-1a8c75ab80a6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:43:09.281Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"1cddad0d-8bd3-4d4f-ad2e-1a8c75ab80a6","seconds":300}
{"level":"info","ts":"2026-06-05T09:43:09.591Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"8841ea9c-3c1a-4a0b-840b-5d1b85cf59c2"}
{"level":"info","ts":"2026-06-05T09:43:09.591Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"8841ea9c-3c1a-4a0b-840b-5d1b85cf59c2"}
{"level":"info","ts":"2026-06-05T09:43:09.591Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"8841ea9c-3c1a-4a0b-840b-5d1b85cf59c2","1 head service found":"counter-4qmws-head-svc"}
{"level":"info","ts":"2026-06-05T09:43:09.591Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"8841ea9c-3c1a-4a0b-840b-5d1b85cf59c2","Found 1 head Pod":"counter-4qmws-head-gfw44","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:43:09.591Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"8841ea9c-3c1a-4a0b-840b-5d1b85cf59c2","head Pod":"counter-4qmws-head-gfw44","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-4qmws-head-gfw44. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:09.591Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"8841ea9c-3c1a-4a0b-840b-5d1b85cf59c2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:43:09.591Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"8841ea9c-3c1a-4a0b-840b-5d1b85cf59c2","worker Pod":"counter-4qmws-small-group-worker-n29x6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-4qmws-small-group-worker-n29x6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:09.591Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"8841ea9c-3c1a-4a0b-840b-5d1b85cf59c2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:09.591Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"8841ea9c-3c1a-4a0b-840b-5d1b85cf59c2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:43:09.591Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"8841ea9c-3c1a-4a0b-840b-5d1b85cf59c2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:09.591Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"8841ea9c-3c1a-4a0b-840b-5d1b85cf59c2","podName":"counter-4qmws-head-gfw44","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:43:09.592Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"8841ea9c-3c1a-4a0b-840b-5d1b85cf59c2","podName":"counter-4qmws-head-gfw44","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:43:09.592Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"8841ea9c-3c1a-4a0b-840b-5d1b85cf59c2","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"counter-4qmws-head-gfw44","serviceName":"counter-4qmws-head-svc"},"newHead":{"podIP":"10.132.0.21","serviceIP":"10.132.0.21","podName":"counter-4qmws-head-gfw44","serviceName":"counter-4qmws-head-svc"}}
{"level":"info","ts":"2026-06-05T09:43:09.592Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"8841ea9c-3c1a-4a0b-840b-5d1b85cf59c2","name":"counter-4qmws","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:43:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-4qmws-head-gfw44","serviceName":"counter-4qmws-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:43:09Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.21","serviceIP":"10.132.0.21","podName":"counter-4qmws-head-gfw44","serviceName":"counter-4qmws-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:43:09.604Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"facca2bc-efc4-4f8a-b854-24e7b5b933bb","cluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:09.604Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"15cca82f-efa1-49f0-9547-a1dc1f40f104","rayCluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:09.604Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"9b228eeb-6d05-4fff-8add-93163163decf","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:09.604Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"15cca82f-efa1-49f0-9547-a1dc1f40f104","pod":"counter-4qmws-head-gfw44","ip":"10.132.0.21"}
{"level":"info","ts":"2026-06-05T09:43:09.604Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"15cca82f-efa1-49f0-9547-a1dc1f40f104","pod":"counter-4qmws-small-group-worker-n29x6","ip":"10.133.0.41"}
{"level":"info","ts":"2026-06-05T09:43:09.604Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"9b228eeb-6d05-4fff-8add-93163163decf","name":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:09.604Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"9b228eeb-6d05-4fff-8add-93163163decf","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:09.604Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"9b228eeb-6d05-4fff-8add-93163163decf"}
{"level":"info","ts":"2026-06-05T09:43:09.604Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"9b228eeb-6d05-4fff-8add-93163163decf","RayCluster":"counter-4qmws","State":""}
{"level":"info","ts":"2026-06-05T09:43:09.615Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"facca2bc-efc4-4f8a-b854-24e7b5b933bb","name":"counter-4qmws-head"}
{"level":"info","ts":"2026-06-05T09:43:09.616Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"15cca82f-efa1-49f0-9547-a1dc1f40f104","certificate":"ray-head-cert-counter-4qmws","dnsNames":["counter-4qmws-head-svc","localhost","counter-4qmws-head-svc.test-ns-t5s4r.svc","counter-4qmws-head-svc.test-ns-t5s4r.svc.cluster.local"],"ipAddresses":["10.132.0.21","10.133.0.41","127.0.0.1"]}
{"level":"info","ts":"2026-06-05T09:43:09.628Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"facca2bc-efc4-4f8a-b854-24e7b5b933bb","name":"counter-4qmws-workers"}
{"level":"info","ts":"2026-06-05T09:43:09.628Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"facca2bc-efc4-4f8a-b854-24e7b5b933bb","cluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:09.628Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"15cca82f-efa1-49f0-9547-a1dc1f40f104","certificate":"ray-worker-cert-counter-4qmws","dnsNames":["counter-4qmws-worker-svc","localhost","counter-4qmws-small-group","counter-4qmws-worker-svc.test-ns-t5s4r.svc","counter-4qmws-worker-svc.test-ns-t5s4r.svc.cluster.local","*.counter-4qmws-worker-svc.test-ns-t5s4r.svc","*.counter-4qmws-worker-svc.test-ns-t5s4r.svc.cluster.local","*-worker-*.test-ns-t5s4r.svc","*-worker-*.test-ns-t5s4r.svc.cluster.local"],"ipAddresses":["10.132.0.21","10.133.0.41","127.0.0.1"]}
{"level":"info","ts":"2026-06-05T09:43:09.628Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"15cca82f-efa1-49f0-9547-a1dc1f40f104","rayCluster":"counter-4qmws","podIPs":["10.132.0.21","10.133.0.41"]}
{"level":"info","ts":"2026-06-05T09:43:10.718Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"b498e284-9c35-40af-bdeb-9a29ff295b64"}
{"level":"info","ts":"2026-06-05T09:43:10.718Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"b498e284-9c35-40af-bdeb-9a29ff295b64"}
{"level":"info","ts":"2026-06-05T09:43:10.718Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"b498e284-9c35-40af-bdeb-9a29ff295b64","1 head service found":"counter-4qmws-head-svc"}
{"level":"info","ts":"2026-06-05T09:43:10.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"b498e284-9c35-40af-bdeb-9a29ff295b64","Found 1 head Pod":"counter-4qmws-head-gfw44","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:43:10.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"b498e284-9c35-40af-bdeb-9a29ff295b64","head Pod":"counter-4qmws-head-gfw44","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-4qmws-head-gfw44. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:10.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"b498e284-9c35-40af-bdeb-9a29ff295b64","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:43:10.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"b498e284-9c35-40af-bdeb-9a29ff295b64","worker Pod":"counter-4qmws-small-group-worker-n29x6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-4qmws-small-group-worker-n29x6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:10.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"b498e284-9c35-40af-bdeb-9a29ff295b64","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:10.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"b498e284-9c35-40af-bdeb-9a29ff295b64","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:43:10.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"b498e284-9c35-40af-bdeb-9a29ff295b64","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:10.719Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"b498e284-9c35-40af-bdeb-9a29ff295b64","podName":"counter-4qmws-head-gfw44","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:43:10.719Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"b498e284-9c35-40af-bdeb-9a29ff295b64","podName":"counter-4qmws-head-gfw44","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:43:10.719Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"b498e284-9c35-40af-bdeb-9a29ff295b64","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:43:10.719Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"b498e284-9c35-40af-bdeb-9a29ff295b64","seconds":300}
{"level":"info","ts":"2026-06-05T09:43:11.523Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"35cee1f9-6000-466c-abf4-9033936244dc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:11.523Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"35cee1f9-6000-466c-abf4-9033936244dc","name":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:11.523Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"35cee1f9-6000-466c-abf4-9033936244dc","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:11.523Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"35cee1f9-6000-466c-abf4-9033936244dc"}
{"level":"info","ts":"2026-06-05T09:43:11.523Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"35cee1f9-6000-466c-abf4-9033936244dc","RayCluster":"counter-4qmws","State":""}
{"level":"info","ts":"2026-06-05T09:43:14.524Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"70ea14ee-56c2-4593-9fc3-c72003d2e5e2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:14.524Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"70ea14ee-56c2-4593-9fc3-c72003d2e5e2","name":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:14.524Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"70ea14ee-56c2-4593-9fc3-c72003d2e5e2","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:14.524Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"70ea14ee-56c2-4593-9fc3-c72003d2e5e2"}
{"level":"info","ts":"2026-06-05T09:43:14.524Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"70ea14ee-56c2-4593-9fc3-c72003d2e5e2","RayCluster":"counter-4qmws","State":""}
{"level":"info","ts":"2026-06-05T09:43:14.611Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"46b411cd-6084-4c3d-9eac-f3a0c8f64c5c"}
{"level":"info","ts":"2026-06-05T09:43:14.611Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"46b411cd-6084-4c3d-9eac-f3a0c8f64c5c"}
{"level":"info","ts":"2026-06-05T09:43:14.611Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"46b411cd-6084-4c3d-9eac-f3a0c8f64c5c","1 head service found":"counter-4qmws-head-svc"}
{"level":"info","ts":"2026-06-05T09:43:14.611Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"46b411cd-6084-4c3d-9eac-f3a0c8f64c5c","Found 1 head Pod":"counter-4qmws-head-gfw44","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:43:14.611Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"46b411cd-6084-4c3d-9eac-f3a0c8f64c5c","head Pod":"counter-4qmws-head-gfw44","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-4qmws-head-gfw44. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:14.611Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"46b411cd-6084-4c3d-9eac-f3a0c8f64c5c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:43:14.611Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"46b411cd-6084-4c3d-9eac-f3a0c8f64c5c","worker Pod":"counter-4qmws-small-group-worker-n29x6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-4qmws-small-group-worker-n29x6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:14.611Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"46b411cd-6084-4c3d-9eac-f3a0c8f64c5c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:14.611Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"46b411cd-6084-4c3d-9eac-f3a0c8f64c5c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:43:14.611Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"46b411cd-6084-4c3d-9eac-f3a0c8f64c5c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:14.611Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"46b411cd-6084-4c3d-9eac-f3a0c8f64c5c","podName":"counter-4qmws-head-gfw44","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:43:14.611Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"46b411cd-6084-4c3d-9eac-f3a0c8f64c5c","podName":"counter-4qmws-head-gfw44","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:43:14.611Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"46b411cd-6084-4c3d-9eac-f3a0c8f64c5c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:43:14.611Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"46b411cd-6084-4c3d-9eac-f3a0c8f64c5c","seconds":300}
{"level":"info","ts":"2026-06-05T09:43:15.611Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"e33713a9-32bc-4a5a-add1-bd9580829da6"}
{"level":"info","ts":"2026-06-05T09:43:15.611Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"e33713a9-32bc-4a5a-add1-bd9580829da6"}
{"level":"info","ts":"2026-06-05T09:43:15.611Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"e33713a9-32bc-4a5a-add1-bd9580829da6","1 head service found":"counter-4qmws-head-svc"}
{"level":"info","ts":"2026-06-05T09:43:15.612Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"e33713a9-32bc-4a5a-add1-bd9580829da6","Found 1 head Pod":"counter-4qmws-head-gfw44","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:43:15.612Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"e33713a9-32bc-4a5a-add1-bd9580829da6","head Pod":"counter-4qmws-head-gfw44","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-4qmws-head-gfw44. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:15.612Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"e33713a9-32bc-4a5a-add1-bd9580829da6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:43:15.612Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"e33713a9-32bc-4a5a-add1-bd9580829da6","worker Pod":"counter-4qmws-small-group-worker-n29x6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-4qmws-small-group-worker-n29x6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:15.612Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"e33713a9-32bc-4a5a-add1-bd9580829da6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:15.612Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"e33713a9-32bc-4a5a-add1-bd9580829da6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:43:15.612Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"e33713a9-32bc-4a5a-add1-bd9580829da6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:15.612Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"e33713a9-32bc-4a5a-add1-bd9580829da6","podName":"counter-4qmws-head-gfw44","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:43:08Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:43:15.612Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"e33713a9-32bc-4a5a-add1-bd9580829da6","podName":"counter-4qmws-head-gfw44","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:43:08Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:43:15.612Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"e33713a9-32bc-4a5a-add1-bd9580829da6","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-05T09:43:15.612Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"e33713a9-32bc-4a5a-add1-bd9580829da6","name":"counter-4qmws","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:43:09Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.21","serviceIP":"10.132.0.21","podName":"counter-4qmws-head-gfw44","serviceName":"counter-4qmws-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:43:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.21","serviceIP":"10.132.0.21","podName":"counter-4qmws-head-gfw44","serviceName":"counter-4qmws-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:43:15.625Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"d8c2c1df-7f7c-4063-ae53-599cfc080f43","cluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:15.625Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"a55514ab-c61d-4f20-bd65-fbc00ac5956f","rayCluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:15.625Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"a4bce320-bb8e-43bb-8fc5-d5d782d6ca2a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:15.625Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"a4bce320-bb8e-43bb-8fc5-d5d782d6ca2a","name":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:15.625Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"a4bce320-bb8e-43bb-8fc5-d5d782d6ca2a","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:15.625Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"a55514ab-c61d-4f20-bd65-fbc00ac5956f","pod":"counter-4qmws-head-gfw44","ip":"10.132.0.21"}
{"level":"info","ts":"2026-06-05T09:43:15.625Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"a55514ab-c61d-4f20-bd65-fbc00ac5956f","pod":"counter-4qmws-small-group-worker-n29x6","ip":"10.133.0.41"}
{"level":"info","ts":"2026-06-05T09:43:15.625Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"a55514ab-c61d-4f20-bd65-fbc00ac5956f","rayCluster":"counter-4qmws","podIPs":["10.132.0.21","10.133.0.41"]}
{"level":"info","ts":"2026-06-05T09:43:15.625Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"a4bce320-bb8e-43bb-8fc5-d5d782d6ca2a"}
{"level":"info","ts":"2026-06-05T09:43:15.625Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"a4bce320-bb8e-43bb-8fc5-d5d782d6ca2a","RayCluster":"counter-4qmws","State":""}
{"level":"info","ts":"2026-06-05T09:43:15.645Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"d8c2c1df-7f7c-4063-ae53-599cfc080f43","name":"counter-4qmws-head"}
{"level":"info","ts":"2026-06-05T09:43:15.655Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"d8c2c1df-7f7c-4063-ae53-599cfc080f43","name":"counter-4qmws-workers"}
{"level":"info","ts":"2026-06-05T09:43:15.655Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"d8c2c1df-7f7c-4063-ae53-599cfc080f43","cluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:17.524Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"a0c31425-28d8-46b5-af84-ace55e3ae2f5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:17.524Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"a0c31425-28d8-46b5-af84-ace55e3ae2f5","name":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:17.525Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"a0c31425-28d8-46b5-af84-ace55e3ae2f5","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:17.525Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"a0c31425-28d8-46b5-af84-ace55e3ae2f5"}
{"level":"info","ts":"2026-06-05T09:43:17.525Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"a0c31425-28d8-46b5-af84-ace55e3ae2f5","RayCluster":"counter-4qmws","State":""}
{"level":"info","ts":"2026-06-05T09:43:17.626Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"eb6b94fa-a8da-42e0-a7f6-f64003158255"}
{"level":"info","ts":"2026-06-05T09:43:17.626Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"eb6b94fa-a8da-42e0-a7f6-f64003158255"}
{"level":"info","ts":"2026-06-05T09:43:17.626Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"eb6b94fa-a8da-42e0-a7f6-f64003158255","1 head service found":"counter-4qmws-head-svc"}
{"level":"info","ts":"2026-06-05T09:43:17.626Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"eb6b94fa-a8da-42e0-a7f6-f64003158255","Found 1 head Pod":"counter-4qmws-head-gfw44","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:43:17.626Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"eb6b94fa-a8da-42e0-a7f6-f64003158255","head Pod":"counter-4qmws-head-gfw44","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-4qmws-head-gfw44. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:17.626Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"eb6b94fa-a8da-42e0-a7f6-f64003158255","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:43:17.626Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"eb6b94fa-a8da-42e0-a7f6-f64003158255","worker Pod":"counter-4qmws-small-group-worker-n29x6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-4qmws-small-group-worker-n29x6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:17.626Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"eb6b94fa-a8da-42e0-a7f6-f64003158255","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:17.626Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"eb6b94fa-a8da-42e0-a7f6-f64003158255","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:43:17.626Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"eb6b94fa-a8da-42e0-a7f6-f64003158255","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:17.626Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"eb6b94fa-a8da-42e0-a7f6-f64003158255","podName":"counter-4qmws-head-gfw44","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:43:08Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:43:17.626Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"eb6b94fa-a8da-42e0-a7f6-f64003158255","podName":"counter-4qmws-head-gfw44","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:43:08Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:43:17.626Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"eb6b94fa-a8da-42e0-a7f6-f64003158255","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:43:17.626Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"eb6b94fa-a8da-42e0-a7f6-f64003158255","seconds":300}
{"level":"info","ts":"2026-06-05T09:43:19.344Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"abe36f29-d782-429a-8b88-43062f4de09f"}
{"level":"info","ts":"2026-06-05T09:43:19.344Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"abe36f29-d782-429a-8b88-43062f4de09f"}
{"level":"info","ts":"2026-06-05T09:43:19.344Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"abe36f29-d782-429a-8b88-43062f4de09f","1 head service found":"counter-4qmws-head-svc"}
{"level":"info","ts":"2026-06-05T09:43:19.344Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"abe36f29-d782-429a-8b88-43062f4de09f","Found 1 head Pod":"counter-4qmws-head-gfw44","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:43:19.344Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"abe36f29-d782-429a-8b88-43062f4de09f","head Pod":"counter-4qmws-head-gfw44","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-4qmws-head-gfw44. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:19.344Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"abe36f29-d782-429a-8b88-43062f4de09f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:43:19.344Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"abe36f29-d782-429a-8b88-43062f4de09f","worker Pod":"counter-4qmws-small-group-worker-n29x6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-4qmws-small-group-worker-n29x6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:19.344Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"abe36f29-d782-429a-8b88-43062f4de09f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:19.344Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"abe36f29-d782-429a-8b88-43062f4de09f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:43:19.344Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"abe36f29-d782-429a-8b88-43062f4de09f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:19.344Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"abe36f29-d782-429a-8b88-43062f4de09f","podName":"counter-4qmws-head-gfw44","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:43:08Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:43:19.344Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"abe36f29-d782-429a-8b88-43062f4de09f","podName":"counter-4qmws-head-gfw44","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:43:08Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:43:19.345Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"abe36f29-d782-429a-8b88-43062f4de09f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:43:19.345Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"abe36f29-d782-429a-8b88-43062f4de09f","seconds":300}
{"level":"info","ts":"2026-06-05T09:43:20.350Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"555358c9-2507-4e55-9e02-e0044bd16887"}
{"level":"info","ts":"2026-06-05T09:43:20.350Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"555358c9-2507-4e55-9e02-e0044bd16887"}
{"level":"info","ts":"2026-06-05T09:43:20.350Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"555358c9-2507-4e55-9e02-e0044bd16887","1 head service found":"counter-4qmws-head-svc"}
{"level":"info","ts":"2026-06-05T09:43:20.350Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"555358c9-2507-4e55-9e02-e0044bd16887","Found 1 head Pod":"counter-4qmws-head-gfw44","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:43:20.350Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"555358c9-2507-4e55-9e02-e0044bd16887","head Pod":"counter-4qmws-head-gfw44","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-4qmws-head-gfw44. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:20.350Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"555358c9-2507-4e55-9e02-e0044bd16887","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:43:20.350Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"555358c9-2507-4e55-9e02-e0044bd16887","worker Pod":"counter-4qmws-small-group-worker-n29x6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-4qmws-small-group-worker-n29x6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:20.350Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"555358c9-2507-4e55-9e02-e0044bd16887","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:20.350Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"555358c9-2507-4e55-9e02-e0044bd16887","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:43:20.350Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"555358c9-2507-4e55-9e02-e0044bd16887","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:20.350Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"555358c9-2507-4e55-9e02-e0044bd16887","podName":"counter-4qmws-head-gfw44","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:43:08Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:43:20.350Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"555358c9-2507-4e55-9e02-e0044bd16887","podName":"counter-4qmws-head-gfw44","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:43:08Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:43:20.351Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"555358c9-2507-4e55-9e02-e0044bd16887","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-05T09:43:20.351Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"555358c9-2507-4e55-9e02-e0044bd16887","name":"counter-4qmws","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:43:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.21","serviceIP":"10.132.0.21","podName":"counter-4qmws-head-gfw44","serviceName":"counter-4qmws-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:43:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.21","serviceIP":"10.132.0.21","podName":"counter-4qmws-head-gfw44","serviceName":"counter-4qmws-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:43:20.363Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"17f0a0a5-56a0-4136-813d-74d0b3da42a9","rayCluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:20.363Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"16ab34e3-d6d6-4ce2-a9bf-ce3928a5f6af","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:20.363Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"16ab34e3-d6d6-4ce2-a9bf-ce3928a5f6af","name":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:20.363Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"16ab34e3-d6d6-4ce2-a9bf-ce3928a5f6af","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:20.363Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"b959e670-fd09-4ce6-b81b-584d61aa2f08","cluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:20.363Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"17f0a0a5-56a0-4136-813d-74d0b3da42a9","pod":"counter-4qmws-small-group-worker-n29x6","ip":"10.133.0.41"}
{"level":"info","ts":"2026-06-05T09:43:20.363Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"17f0a0a5-56a0-4136-813d-74d0b3da42a9","pod":"counter-4qmws-head-gfw44","ip":"10.132.0.21"}
{"level":"info","ts":"2026-06-05T09:43:20.363Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"16ab34e3-d6d6-4ce2-a9bf-ce3928a5f6af"}
{"level":"info","ts":"2026-06-05T09:43:20.363Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"16ab34e3-d6d6-4ce2-a9bf-ce3928a5f6af","RayCluster":"counter-4qmws","State":""}
{"level":"info","ts":"2026-06-05T09:43:20.363Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"17f0a0a5-56a0-4136-813d-74d0b3da42a9","rayCluster":"counter-4qmws","podIPs":["10.133.0.41","10.132.0.21"]}
{"level":"info","ts":"2026-06-05T09:43:20.373Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"b959e670-fd09-4ce6-b81b-584d61aa2f08","name":"counter-4qmws-head"}
{"level":"info","ts":"2026-06-05T09:43:20.383Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"b959e670-fd09-4ce6-b81b-584d61aa2f08","name":"counter-4qmws-workers"}
{"level":"info","ts":"2026-06-05T09:43:20.383Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"b959e670-fd09-4ce6-b81b-584d61aa2f08","cluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:20.525Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"ca9b3bea-7160-477b-b9f2-60c73c22ca22","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:20.525Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"ca9b3bea-7160-477b-b9f2-60c73c22ca22","name":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:20.525Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"ca9b3bea-7160-477b-b9f2-60c73c22ca22","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:20.526Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"ca9b3bea-7160-477b-b9f2-60c73c22ca22"}
{"level":"info","ts":"2026-06-05T09:43:20.526Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"ca9b3bea-7160-477b-b9f2-60c73c22ca22","RayCluster":"counter-4qmws","State":""}
{"level":"info","ts":"2026-06-05T09:43:22.363Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"744f3ea9-a565-4fbe-8e9b-3b6a266d59aa"}
{"level":"info","ts":"2026-06-05T09:43:22.363Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"744f3ea9-a565-4fbe-8e9b-3b6a266d59aa"}
{"level":"info","ts":"2026-06-05T09:43:22.363Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"744f3ea9-a565-4fbe-8e9b-3b6a266d59aa","1 head service found":"counter-4qmws-head-svc"}
{"level":"info","ts":"2026-06-05T09:43:22.363Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"744f3ea9-a565-4fbe-8e9b-3b6a266d59aa","Found 1 head Pod":"counter-4qmws-head-gfw44","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:43:22.363Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"744f3ea9-a565-4fbe-8e9b-3b6a266d59aa","head Pod":"counter-4qmws-head-gfw44","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-4qmws-head-gfw44. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:22.363Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"744f3ea9-a565-4fbe-8e9b-3b6a266d59aa","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:43:22.364Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"744f3ea9-a565-4fbe-8e9b-3b6a266d59aa","worker Pod":"counter-4qmws-small-group-worker-n29x6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-4qmws-small-group-worker-n29x6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:22.364Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"744f3ea9-a565-4fbe-8e9b-3b6a266d59aa","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:22.364Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"744f3ea9-a565-4fbe-8e9b-3b6a266d59aa","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:43:22.364Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"744f3ea9-a565-4fbe-8e9b-3b6a266d59aa","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:22.364Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"744f3ea9-a565-4fbe-8e9b-3b6a266d59aa","podName":"counter-4qmws-head-gfw44","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:43:08Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:43:22.364Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"744f3ea9-a565-4fbe-8e9b-3b6a266d59aa","podName":"counter-4qmws-head-gfw44","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:43:08Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:43:22.364Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"744f3ea9-a565-4fbe-8e9b-3b6a266d59aa","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:43:22.364Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"744f3ea9-a565-4fbe-8e9b-3b6a266d59aa","seconds":300}
{"level":"info","ts":"2026-06-05T09:43:23.526Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"3a2e4ec4-287a-4290-a5ac-2a9c1c590256","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:23.526Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"3a2e4ec4-287a-4290-a5ac-2a9c1c590256","name":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:23.526Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"3a2e4ec4-287a-4290-a5ac-2a9c1c590256","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:23.526Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"3a2e4ec4-287a-4290-a5ac-2a9c1c590256"}
{"level":"info","ts":"2026-06-05T09:43:23.526Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"3a2e4ec4-287a-4290-a5ac-2a9c1c590256","RayCluster":"counter-4qmws","State":""}
{"level":"info","ts":"2026-06-05T09:43:26.527Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"a1409e98-0c08-4414-ac71-b36a9d8081d8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:26.527Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"a1409e98-0c08-4414-ac71-b36a9d8081d8","name":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:26.527Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"a1409e98-0c08-4414-ac71-b36a9d8081d8","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:26.527Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"a1409e98-0c08-4414-ac71-b36a9d8081d8"}
{"level":"info","ts":"2026-06-05T09:43:26.527Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"a1409e98-0c08-4414-ac71-b36a9d8081d8","RayCluster":"counter-4qmws","State":""}
{"level":"info","ts":"2026-06-05T09:43:26.640Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"209a137b-b967-4738-8734-2b5b409ccd7e"}
{"level":"info","ts":"2026-06-05T09:43:26.640Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"209a137b-b967-4738-8734-2b5b409ccd7e"}
{"level":"info","ts":"2026-06-05T09:43:26.640Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"209a137b-b967-4738-8734-2b5b409ccd7e","1 head service found":"counter-4qmws-head-svc"}
{"level":"info","ts":"2026-06-05T09:43:26.640Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"209a137b-b967-4738-8734-2b5b409ccd7e","Found 1 head Pod":"counter-4qmws-head-gfw44","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:43:26.640Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"209a137b-b967-4738-8734-2b5b409ccd7e","head Pod":"counter-4qmws-head-gfw44","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-4qmws-head-gfw44. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:26.640Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"209a137b-b967-4738-8734-2b5b409ccd7e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:43:26.640Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"209a137b-b967-4738-8734-2b5b409ccd7e","worker Pod":"counter-4qmws-small-group-worker-n29x6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-4qmws-small-group-worker-n29x6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:26.640Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"209a137b-b967-4738-8734-2b5b409ccd7e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:26.640Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"209a137b-b967-4738-8734-2b5b409ccd7e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:43:26.640Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"209a137b-b967-4738-8734-2b5b409ccd7e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:26.640Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"209a137b-b967-4738-8734-2b5b409ccd7e","podName":"counter-4qmws-small-group-worker-n29x6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:43:08Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-05T09:43:26.640Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"209a137b-b967-4738-8734-2b5b409ccd7e","podName":"counter-4qmws-small-group-worker-n29x6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:43:08Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-05T09:43:26.640Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"209a137b-b967-4738-8734-2b5b409ccd7e","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:43:26Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-05T09:43:26.640Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"209a137b-b967-4738-8734-2b5b409ccd7e","name":"counter-4qmws","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:43:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.21","serviceIP":"10.132.0.21","podName":"counter-4qmws-head-gfw44","serviceName":"counter-4qmws-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:43:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.21","serviceIP":"10.132.0.21","podName":"counter-4qmws-head-gfw44","serviceName":"counter-4qmws-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:43:26Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:43:26.652Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"8bbb3036-dccb-4f31-82e3-1c4578b5a611","rayCluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:26.652Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"1cd119a8-da21-425b-937a-6fb03d0d0024","cluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:26.652Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"8bbb3036-dccb-4f31-82e3-1c4578b5a611","pod":"counter-4qmws-head-gfw44","ip":"10.132.0.21"}
{"level":"info","ts":"2026-06-05T09:43:26.652Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"8bbb3036-dccb-4f31-82e3-1c4578b5a611","pod":"counter-4qmws-small-group-worker-n29x6","ip":"10.133.0.41"}
{"level":"info","ts":"2026-06-05T09:43:26.652Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"8bbb3036-dccb-4f31-82e3-1c4578b5a611","rayCluster":"counter-4qmws","podIPs":["10.132.0.21","10.133.0.41"]}
{"level":"info","ts":"2026-06-05T09:43:26.652Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"d861eee2-e9d6-4763-9c3e-5d48f33af86c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:26.652Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"d861eee2-e9d6-4763-9c3e-5d48f33af86c","name":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:26.652Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"d861eee2-e9d6-4763-9c3e-5d48f33af86c","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:26.652Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"d861eee2-e9d6-4763-9c3e-5d48f33af86c"}
{"level":"info","ts":"2026-06-05T09:43:26.652Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"d861eee2-e9d6-4763-9c3e-5d48f33af86c","RayCluster":"counter-4qmws","State":""}
{"level":"info","ts":"2026-06-05T09:43:26.665Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"1cd119a8-da21-425b-937a-6fb03d0d0024","name":"counter-4qmws-head"}
{"level":"info","ts":"2026-06-05T09:43:26.675Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"1cd119a8-da21-425b-937a-6fb03d0d0024","name":"counter-4qmws-workers"}
{"level":"info","ts":"2026-06-05T09:43:26.675Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"1cd119a8-da21-425b-937a-6fb03d0d0024","cluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:28.652Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"b3a3bd37-53f7-4b12-86dd-2355982215ec"}
{"level":"info","ts":"2026-06-05T09:43:28.652Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"b3a3bd37-53f7-4b12-86dd-2355982215ec"}
{"level":"info","ts":"2026-06-05T09:43:28.652Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"b3a3bd37-53f7-4b12-86dd-2355982215ec","1 head service found":"counter-4qmws-head-svc"}
{"level":"info","ts":"2026-06-05T09:43:28.653Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"b3a3bd37-53f7-4b12-86dd-2355982215ec","Found 1 head Pod":"counter-4qmws-head-gfw44","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:43:28.653Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"b3a3bd37-53f7-4b12-86dd-2355982215ec","head Pod":"counter-4qmws-head-gfw44","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-4qmws-head-gfw44. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:28.653Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"b3a3bd37-53f7-4b12-86dd-2355982215ec","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:43:28.653Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"b3a3bd37-53f7-4b12-86dd-2355982215ec","worker Pod":"counter-4qmws-small-group-worker-n29x6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-4qmws-small-group-worker-n29x6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:28.653Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"b3a3bd37-53f7-4b12-86dd-2355982215ec","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:28.653Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"b3a3bd37-53f7-4b12-86dd-2355982215ec","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:43:28.653Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"b3a3bd37-53f7-4b12-86dd-2355982215ec","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:28.653Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"b3a3bd37-53f7-4b12-86dd-2355982215ec","podName":"counter-4qmws-small-group-worker-n29x6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:43:08Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-05T09:43:28.653Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"b3a3bd37-53f7-4b12-86dd-2355982215ec","podName":"counter-4qmws-small-group-worker-n29x6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:43:08Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-05T09:43:28.653Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"b3a3bd37-53f7-4b12-86dd-2355982215ec","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:43:28.653Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"b3a3bd37-53f7-4b12-86dd-2355982215ec","seconds":300}
{"level":"info","ts":"2026-06-05T09:43:29.528Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"3ea4d8b5-6185-4e2a-8238-d54e04adaaf8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:29.528Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"3ea4d8b5-6185-4e2a-8238-d54e04adaaf8","name":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:29.528Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"3ea4d8b5-6185-4e2a-8238-d54e04adaaf8","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:29.528Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"3ea4d8b5-6185-4e2a-8238-d54e04adaaf8"}
{"level":"info","ts":"2026-06-05T09:43:29.528Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"3ea4d8b5-6185-4e2a-8238-d54e04adaaf8","RayCluster":"counter-4qmws","State":""}
{"level":"info","ts":"2026-06-05T09:43:31.375Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"dfa2f46e-fa04-45a6-a2bf-0b871bb85fe8"}
{"level":"info","ts":"2026-06-05T09:43:31.375Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"dfa2f46e-fa04-45a6-a2bf-0b871bb85fe8"}
{"level":"info","ts":"2026-06-05T09:43:31.375Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"dfa2f46e-fa04-45a6-a2bf-0b871bb85fe8","1 head service found":"counter-4qmws-head-svc"}
{"level":"info","ts":"2026-06-05T09:43:31.375Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"dfa2f46e-fa04-45a6-a2bf-0b871bb85fe8","Found 1 head Pod":"counter-4qmws-head-gfw44","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:43:31.375Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"dfa2f46e-fa04-45a6-a2bf-0b871bb85fe8","head Pod":"counter-4qmws-head-gfw44","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-4qmws-head-gfw44. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:31.375Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"dfa2f46e-fa04-45a6-a2bf-0b871bb85fe8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:43:31.375Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"dfa2f46e-fa04-45a6-a2bf-0b871bb85fe8","worker Pod":"counter-4qmws-small-group-worker-n29x6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-4qmws-small-group-worker-n29x6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:31.375Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"dfa2f46e-fa04-45a6-a2bf-0b871bb85fe8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:31.375Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"dfa2f46e-fa04-45a6-a2bf-0b871bb85fe8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:43:31.375Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"dfa2f46e-fa04-45a6-a2bf-0b871bb85fe8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:31.375Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"dfa2f46e-fa04-45a6-a2bf-0b871bb85fe8","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-06-05T09:43:31.375Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"dfa2f46e-fa04-45a6-a2bf-0b871bb85fe8","name":"counter-4qmws","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:43:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.21","serviceIP":"10.132.0.21","podName":"counter-4qmws-head-gfw44","serviceName":"counter-4qmws-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:43:26Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:43:31Z","stateTransitionTimes":{"ready":"2026-06-05T09:43:31Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.21","serviceIP":"10.132.0.21","podName":"counter-4qmws-head-gfw44","serviceName":"counter-4qmws-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:43:26Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:43:31Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:43:31.388Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"859f7d77-f464-497f-9ba1-0ba3671097e5","cluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:31.388Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"58700671-f605-461b-80e7-73c44e3620b2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:31.388Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"58700671-f605-461b-80e7-73c44e3620b2","name":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:31.388Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"58700671-f605-461b-80e7-73c44e3620b2","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:31.388Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"d9e718c4-92a3-4e35-a5d9-d9d5356b75d7","rayCluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:31.388Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"58700671-f605-461b-80e7-73c44e3620b2"}
{"level":"info","ts":"2026-06-05T09:43:31.388Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"d9e718c4-92a3-4e35-a5d9-d9d5356b75d7","pod":"counter-4qmws-head-gfw44","ip":"10.132.0.21"}
{"level":"info","ts":"2026-06-05T09:43:31.388Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"d9e718c4-92a3-4e35-a5d9-d9d5356b75d7","pod":"counter-4qmws-small-group-worker-n29x6","ip":"10.133.0.41"}
{"level":"info","ts":"2026-06-05T09:43:31.388Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"58700671-f605-461b-80e7-73c44e3620b2","head service name":"counter-4qmws-head-svc","namespace":"test-ns-t5s4r"}
{"level":"info","ts":"2026-06-05T09:43:31.388Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"58700671-f605-461b-80e7-73c44e3620b2","head service URL":"counter-4qmws-head-svc.test-ns-t5s4r.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-05T09:43:31.388Z","logger":"controllers.RayJob","msg":"user-provided submitter template is used; the first container is assumed to be the submitter","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"58700671-f605-461b-80e7-73c44e3620b2"}
{"level":"info","ts":"2026-06-05T09:43:31.388Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"d9e718c4-92a3-4e35-a5d9-d9d5356b75d7","rayCluster":"counter-4qmws","podIPs":["10.132.0.21","10.133.0.41"]}
{"level":"info","ts":"2026-06-05T09:43:31.389Z","logger":"controllers.RayJob","msg":"No command is specified in the user-provided template. Default command is used","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"58700671-f605-461b-80e7-73c44e3620b2","command":["if","!","ray","job","status","--address","http://counter-4qmws-head-svc.test-ns-t5s4r.svc.cluster.local:8265","counter-dgn8t",">/dev/null","2>&1",";","then","ray","job","submit","--address","http://counter-4qmws-head-svc.test-ns-t5s4r.svc.cluster.local:8265","--no-wait","--runtime-env-json","\"{\\\"env_vars\\\":{\\\"counter_name\\\":\\\"test_counter\\\"}}\"","--submission-id","counter-dgn8t","--","python /home/ray/jobs/counter.py",";","fi",";","ray","job","logs","--address","http://counter-4qmws-head-svc.test-ns-t5s4r.svc.cluster.local:8265","--follow","counter-dgn8t"]}
{"level":"info","ts":"2026-06-05T09:43:31.397Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"58700671-f605-461b-80e7-73c44e3620b2","Kubernetes Job":"counter"}
{"level":"info","ts":"2026-06-05T09:43:31.397Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"58700671-f605-461b-80e7-73c44e3620b2","SubmissionMode":"K8sJobMode","RayCluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:31.397Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"58700671-f605-461b-80e7-73c44e3620b2","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dgn8t","rayClusterName":"counter-4qmws","jobDeploymentStatus":"Initializing","startTime":"2026-06-05T09:43:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dgn8t","rayClusterName":"counter-4qmws","dashboardURL":"counter-4qmws-head-svc.test-ns-t5s4r.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-05T09:43:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:43:31.397Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"58700671-f605-461b-80e7-73c44e3620b2","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-05T09:43:31.402Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"859f7d77-f464-497f-9ba1-0ba3671097e5","name":"counter-4qmws-head"}
{"level":"info","ts":"2026-06-05T09:43:31.408Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"0196dfec-3c69-47cc-8230-c433bc2a2ab4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:31.408Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"0196dfec-3c69-47cc-8230-c433bc2a2ab4","name":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:31.408Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"0196dfec-3c69-47cc-8230-c433bc2a2ab4","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:31.408Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"0196dfec-3c69-47cc-8230-c433bc2a2ab4"}
{"level":"info","ts":"2026-06-05T09:43:31.408Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"0196dfec-3c69-47cc-8230-c433bc2a2ab4","head service name":"counter-4qmws-head-svc","namespace":"test-ns-t5s4r"}
{"level":"info","ts":"2026-06-05T09:43:31.408Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"0196dfec-3c69-47cc-8230-c433bc2a2ab4","head service URL":"counter-4qmws-head-svc.test-ns-t5s4r.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-05T09:43:31.408Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job for RayJob already exists","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"0196dfec-3c69-47cc-8230-c433bc2a2ab4","Kubernetes Job":"counter"}
{"level":"info","ts":"2026-06-05T09:43:31.408Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"0196dfec-3c69-47cc-8230-c433bc2a2ab4","SubmissionMode":"K8sJobMode","RayCluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:31.408Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"0196dfec-3c69-47cc-8230-c433bc2a2ab4","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dgn8t","rayClusterName":"counter-4qmws","jobDeploymentStatus":"Initializing","startTime":"2026-06-05T09:43:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dgn8t","rayClusterName":"counter-4qmws","dashboardURL":"counter-4qmws-head-svc.test-ns-t5s4r.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-05T09:43:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:43:31.408Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"0196dfec-3c69-47cc-8230-c433bc2a2ab4","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-05T09:43:31.416Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"859f7d77-f464-497f-9ba1-0ba3671097e5","name":"counter-4qmws-workers"}
{"level":"info","ts":"2026-06-05T09:43:31.416Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"859f7d77-f464-497f-9ba1-0ba3671097e5","cluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:31.420Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"0196dfec-3c69-47cc-8230-c433bc2a2ab4","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-06-05T09:43:31.420Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"0196dfec-3c69-47cc-8230-c433bc2a2ab4"}
{"level":"error","ts":"2026-06-05T09:43:31.420Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"0196dfec-3c69-47cc-8230-c433bc2a2ab4","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:31.420Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"6ba55a54-1d6a-41cc-8925-96521ffe2342","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:31.420Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"6ba55a54-1d6a-41cc-8925-96521ffe2342","name":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:31.420Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"6ba55a54-1d6a-41cc-8925-96521ffe2342","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:31.420Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"6ba55a54-1d6a-41cc-8925-96521ffe2342"}
{"level":"error","ts":"2026-06-05T09:43:31.429Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"6ba55a54-1d6a-41cc-8925-96521ffe2342","JobId":"counter-dgn8t","error":"Job counter-dgn8t does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:31.430Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"6ba55a54-1d6a-41cc-8925-96521ffe2342"}
{"level":"error","ts":"2026-06-05T09:43:31.430Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"6ba55a54-1d6a-41cc-8925-96521ffe2342","error":"Job counter-dgn8t does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:31.430Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"bf8ba9d8-89a2-4144-869a-696225a04111","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:31.430Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"bf8ba9d8-89a2-4144-869a-696225a04111","name":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:31.430Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"bf8ba9d8-89a2-4144-869a-696225a04111","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:31.430Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"bf8ba9d8-89a2-4144-869a-696225a04111"}
{"level":"error","ts":"2026-06-05T09:43:31.436Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"bf8ba9d8-89a2-4144-869a-696225a04111","JobId":"counter-dgn8t","error":"Job counter-dgn8t does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:31.436Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"bf8ba9d8-89a2-4144-869a-696225a04111"}
{"level":"error","ts":"2026-06-05T09:43:31.436Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"bf8ba9d8-89a2-4144-869a-696225a04111","error":"Job counter-dgn8t does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:31.440Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"e7039e26-ccec-4775-aadc-9b2b77089b4d","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:31.440Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"e7039e26-ccec-4775-aadc-9b2b77089b4d","name":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:31.440Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"e7039e26-ccec-4775-aadc-9b2b77089b4d","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:31.440Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"e7039e26-ccec-4775-aadc-9b2b77089b4d"}
{"level":"error","ts":"2026-06-05T09:43:31.445Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"e7039e26-ccec-4775-aadc-9b2b77089b4d","JobId":"counter-dgn8t","error":"Job counter-dgn8t does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:31.445Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"e7039e26-ccec-4775-aadc-9b2b77089b4d"}
{"level":"error","ts":"2026-06-05T09:43:31.445Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"e7039e26-ccec-4775-aadc-9b2b77089b4d","error":"Job counter-dgn8t does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:31.486Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"179ba874-f745-4868-9360-d33eb5d93bd5","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:31.486Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"179ba874-f745-4868-9360-d33eb5d93bd5","name":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:31.486Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"179ba874-f745-4868-9360-d33eb5d93bd5","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:31.506Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"179ba874-f745-4868-9360-d33eb5d93bd5"}
{"level":"error","ts":"2026-06-05T09:43:31.511Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"179ba874-f745-4868-9360-d33eb5d93bd5","JobId":"counter-dgn8t","error":"Job counter-dgn8t does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:31.511Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"179ba874-f745-4868-9360-d33eb5d93bd5"}
{"level":"error","ts":"2026-06-05T09:43:31.511Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"179ba874-f745-4868-9360-d33eb5d93bd5","error":"Job counter-dgn8t does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:31.591Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"3dea58fb-8bf1-47e2-bebd-df5b8c574740","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:31.592Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"3dea58fb-8bf1-47e2-bebd-df5b8c574740","name":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:31.592Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"3dea58fb-8bf1-47e2-bebd-df5b8c574740","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:31.592Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"3dea58fb-8bf1-47e2-bebd-df5b8c574740"}
{"level":"error","ts":"2026-06-05T09:43:31.709Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"3dea58fb-8bf1-47e2-bebd-df5b8c574740","JobId":"counter-dgn8t","error":"Job counter-dgn8t does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:31.710Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"3dea58fb-8bf1-47e2-bebd-df5b8c574740"}
{"level":"error","ts":"2026-06-05T09:43:31.710Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"3dea58fb-8bf1-47e2-bebd-df5b8c574740","error":"Job counter-dgn8t does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:31.870Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"f69badc6-5ca1-4325-9e68-5ee36a97b3f5","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:31.870Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"f69badc6-5ca1-4325-9e68-5ee36a97b3f5","name":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:31.870Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"f69badc6-5ca1-4325-9e68-5ee36a97b3f5","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:31.870Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"f69badc6-5ca1-4325-9e68-5ee36a97b3f5"}
{"level":"error","ts":"2026-06-05T09:43:31.876Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"f69badc6-5ca1-4325-9e68-5ee36a97b3f5","JobId":"counter-dgn8t","error":"Job counter-dgn8t does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:31.876Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"f69badc6-5ca1-4325-9e68-5ee36a97b3f5"}
{"level":"error","ts":"2026-06-05T09:43:31.876Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"f69badc6-5ca1-4325-9e68-5ee36a97b3f5","error":"Job counter-dgn8t does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:32.197Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"ace04329-72f5-4125-a47f-88d860fcb77c","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:32.197Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"ace04329-72f5-4125-a47f-88d860fcb77c","name":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:32.197Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"ace04329-72f5-4125-a47f-88d860fcb77c","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:32.197Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"ace04329-72f5-4125-a47f-88d860fcb77c"}
{"level":"error","ts":"2026-06-05T09:43:32.202Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"ace04329-72f5-4125-a47f-88d860fcb77c","JobId":"counter-dgn8t","error":"Job counter-dgn8t does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:32.202Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"ace04329-72f5-4125-a47f-88d860fcb77c"}
{"level":"error","ts":"2026-06-05T09:43:32.202Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"ace04329-72f5-4125-a47f-88d860fcb77c","error":"Job counter-dgn8t does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:32.420Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"9f0e3a3a-145f-42e7-80c7-28f307f89492","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:32.420Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"9f0e3a3a-145f-42e7-80c7-28f307f89492","name":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:32.420Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"9f0e3a3a-145f-42e7-80c7-28f307f89492","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:32.420Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"9f0e3a3a-145f-42e7-80c7-28f307f89492"}
{"level":"error","ts":"2026-06-05T09:43:32.425Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"9f0e3a3a-145f-42e7-80c7-28f307f89492","JobId":"counter-dgn8t","error":"Job counter-dgn8t does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:32.425Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"9f0e3a3a-145f-42e7-80c7-28f307f89492"}
{"level":"error","ts":"2026-06-05T09:43:32.425Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"9f0e3a3a-145f-42e7-80c7-28f307f89492","error":"Job counter-dgn8t does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:32.843Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"2fdac00a-b62f-4ed1-aacc-484916e81f82","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:32.843Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"2fdac00a-b62f-4ed1-aacc-484916e81f82","name":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:32.843Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"2fdac00a-b62f-4ed1-aacc-484916e81f82","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:32.843Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"2fdac00a-b62f-4ed1-aacc-484916e81f82"}
{"level":"error","ts":"2026-06-05T09:43:32.848Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"2fdac00a-b62f-4ed1-aacc-484916e81f82","JobId":"counter-dgn8t","error":"Job counter-dgn8t does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:32.848Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"2fdac00a-b62f-4ed1-aacc-484916e81f82"}
{"level":"error","ts":"2026-06-05T09:43:32.848Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"2fdac00a-b62f-4ed1-aacc-484916e81f82","error":"Job counter-dgn8t does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:33.388Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"97e4b07e-d56a-4809-bd85-efde5722b13d"}
{"level":"info","ts":"2026-06-05T09:43:33.388Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"97e4b07e-d56a-4809-bd85-efde5722b13d"}
{"level":"info","ts":"2026-06-05T09:43:33.389Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"97e4b07e-d56a-4809-bd85-efde5722b13d","1 head service found":"counter-4qmws-head-svc"}
{"level":"info","ts":"2026-06-05T09:43:33.389Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"97e4b07e-d56a-4809-bd85-efde5722b13d","Found 1 head Pod":"counter-4qmws-head-gfw44","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:43:33.389Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"97e4b07e-d56a-4809-bd85-efde5722b13d","head Pod":"counter-4qmws-head-gfw44","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-4qmws-head-gfw44. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:33.389Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"97e4b07e-d56a-4809-bd85-efde5722b13d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:43:33.389Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"97e4b07e-d56a-4809-bd85-efde5722b13d","worker Pod":"counter-4qmws-small-group-worker-n29x6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-4qmws-small-group-worker-n29x6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:33.389Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"97e4b07e-d56a-4809-bd85-efde5722b13d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:33.389Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"97e4b07e-d56a-4809-bd85-efde5722b13d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:43:33.389Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"97e4b07e-d56a-4809-bd85-efde5722b13d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:33.389Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"97e4b07e-d56a-4809-bd85-efde5722b13d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:43:33.389Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"97e4b07e-d56a-4809-bd85-efde5722b13d","seconds":300}
{"level":"info","ts":"2026-06-05T09:43:33.607Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"ce73a5ec-0c24-4ae2-adae-1c4acb46a9a4","rayCluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:33.607Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"ce73a5ec-0c24-4ae2-adae-1c4acb46a9a4","pod":"counter-4qmws-small-group-worker-n29x6","ip":"10.133.0.41"}
{"level":"info","ts":"2026-06-05T09:43:33.607Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"ce73a5ec-0c24-4ae2-adae-1c4acb46a9a4","pod":"counter-4qmws-head-gfw44","ip":"10.132.0.21"}
{"level":"info","ts":"2026-06-05T09:43:33.607Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"ce73a5ec-0c24-4ae2-adae-1c4acb46a9a4","rayCluster":"counter-4qmws","podIPs":["10.133.0.41","10.132.0.21"]}
{"level":"info","ts":"2026-06-05T09:43:35.409Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"44734feb-638c-48cc-9479-2ad98983d409","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:35.409Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"44734feb-638c-48cc-9479-2ad98983d409","name":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:35.409Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"44734feb-638c-48cc-9479-2ad98983d409","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:35.409Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"44734feb-638c-48cc-9479-2ad98983d409"}
{"level":"error","ts":"2026-06-05T09:43:35.414Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"44734feb-638c-48cc-9479-2ad98983d409","JobId":"counter-dgn8t","error":"Job counter-dgn8t does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:35.414Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"44734feb-638c-48cc-9479-2ad98983d409"}
{"level":"error","ts":"2026-06-05T09:43:35.414Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"44734feb-638c-48cc-9479-2ad98983d409","error":"Job counter-dgn8t does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:36.551Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"def91825-7b07-4c2c-9fe7-3e461db821e6","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:43:36.552Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"def91825-7b07-4c2c-9fe7-3e461db821e6","pod":"raycluster-head-vjd7d","ip":"10.134.0.20"}
{"level":"info","ts":"2026-06-05T09:43:36.552Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"def91825-7b07-4c2c-9fe7-3e461db821e6","pod":"raycluster-small-group-worker-j9pbw","ip":"10.133.0.35"}
{"level":"info","ts":"2026-06-05T09:43:36.552Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"def91825-7b07-4c2c-9fe7-3e461db821e6","rayCluster":"raycluster","podIPs":["10.134.0.20","10.133.0.35"]}
{"level":"info","ts":"2026-06-05T09:43:40.534Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"37016257-1d6c-4f25-b8bf-3e791cb9a35e","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:40.534Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"37016257-1d6c-4f25-b8bf-3e791cb9a35e","name":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:40.535Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"37016257-1d6c-4f25-b8bf-3e791cb9a35e","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:40.535Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"37016257-1d6c-4f25-b8bf-3e791cb9a35e"}
{"level":"info","ts":"2026-06-05T09:43:40.541Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"37016257-1d6c-4f25-b8bf-3e791cb9a35e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dgn8t","rayClusterName":"counter-4qmws","dashboardURL":"counter-4qmws-head-svc.test-ns-t5s4r.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-05T09:43:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:43:39Z"},"jobId":"counter-dgn8t","rayClusterName":"counter-4qmws","dashboardURL":"counter-4qmws-head-svc.test-ns-t5s4r.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-06-05T09:43:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:43:31Z","stateTransitionTimes":{"ready":"2026-06-05T09:43:31Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.21","serviceIP":"10.132.0.21","podName":"counter-4qmws-head-gfw44","serviceName":"counter-4qmws-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:43:26Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:43:31Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:43:40.541Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"37016257-1d6c-4f25-b8bf-3e791cb9a35e","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-05T09:43:40.556Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"76d54c40-218e-4ee0-ae03-5d4d67d53047","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:40.556Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"76d54c40-218e-4ee0-ae03-5d4d67d53047","name":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:40.556Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"76d54c40-218e-4ee0-ae03-5d4d67d53047","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:40.556Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"76d54c40-218e-4ee0-ae03-5d4d67d53047"}
{"level":"info","ts":"2026-06-05T09:43:40.559Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"76d54c40-218e-4ee0-ae03-5d4d67d53047","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:43:39Z"},"jobId":"counter-dgn8t","rayClusterName":"counter-4qmws","dashboardURL":"counter-4qmws-head-svc.test-ns-t5s4r.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-06-05T09:43:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:43:31Z","stateTransitionTimes":{"ready":"2026-06-05T09:43:31Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.21","serviceIP":"10.132.0.21","podName":"counter-4qmws-head-gfw44","serviceName":"counter-4qmws-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:43:26Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:43:31Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:43:39Z"},"jobId":"counter-dgn8t","rayClusterName":"counter-4qmws","dashboardURL":"counter-4qmws-head-svc.test-ns-t5s4r.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-06-05T09:43:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:43:31Z","stateTransitionTimes":{"ready":"2026-06-05T09:43:31Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.21","serviceIP":"10.132.0.21","podName":"counter-4qmws-head-gfw44","serviceName":"counter-4qmws-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:43:26Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:43:31Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:43:43.556Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"4ef6e741-e4a0-4c59-bd05-adbb94623a45","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:43.556Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"4ef6e741-e4a0-4c59-bd05-adbb94623a45","name":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:43.556Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"4ef6e741-e4a0-4c59-bd05-adbb94623a45","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:43.556Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"4ef6e741-e4a0-4c59-bd05-adbb94623a45"}
{"level":"info","ts":"2026-06-05T09:43:43.559Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"4ef6e741-e4a0-4c59-bd05-adbb94623a45","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:43:39Z"},"jobId":"counter-dgn8t","rayClusterName":"counter-4qmws","dashboardURL":"counter-4qmws-head-svc.test-ns-t5s4r.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-06-05T09:43:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:43:31Z","stateTransitionTimes":{"ready":"2026-06-05T09:43:31Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.21","serviceIP":"10.132.0.21","podName":"counter-4qmws-head-gfw44","serviceName":"counter-4qmws-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:43:26Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:43:31Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:43:39Z","endTime":"2026-06-05T09:43:43Z"},"jobId":"counter-dgn8t","rayClusterName":"counter-4qmws","dashboardURL":"counter-4qmws-head-svc.test-ns-t5s4r.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-05T09:43:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:43:31Z","stateTransitionTimes":{"ready":"2026-06-05T09:43:31Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.21","serviceIP":"10.132.0.21","podName":"counter-4qmws-head-gfw44","serviceName":"counter-4qmws-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:43:26Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:43:31Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:43:43.559Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"4ef6e741-e4a0-4c59-bd05-adbb94623a45","old JobStatus":"PENDING","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-05T09:43:43.574Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"c441acab-127b-458e-bd0c-53a2f904747f","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:43.574Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"c441acab-127b-458e-bd0c-53a2f904747f","name":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:43.574Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"c441acab-127b-458e-bd0c-53a2f904747f","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:43.574Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"c441acab-127b-458e-bd0c-53a2f904747f"}
{"level":"info","ts":"2026-06-05T09:43:43.576Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"c441acab-127b-458e-bd0c-53a2f904747f","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:43:39Z","endTime":"2026-06-05T09:43:43Z"},"jobId":"counter-dgn8t","rayClusterName":"counter-4qmws","dashboardURL":"counter-4qmws-head-svc.test-ns-t5s4r.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-05T09:43:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:43:31Z","stateTransitionTimes":{"ready":"2026-06-05T09:43:31Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.21","serviceIP":"10.132.0.21","podName":"counter-4qmws-head-gfw44","serviceName":"counter-4qmws-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:43:26Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:43:31Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:43:39Z","endTime":"2026-06-05T09:43:43Z"},"jobId":"counter-dgn8t","rayClusterName":"counter-4qmws","dashboardURL":"counter-4qmws-head-svc.test-ns-t5s4r.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-05T09:43:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:43:31Z","stateTransitionTimes":{"ready":"2026-06-05T09:43:31Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.21","serviceIP":"10.132.0.21","podName":"counter-4qmws-head-gfw44","serviceName":"counter-4qmws-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:43:26Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:43:31Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:43:46.574Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"3de991df-a817-4f5b-aaa1-15390af1eec9","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:46.574Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"3de991df-a817-4f5b-aaa1-15390af1eec9","name":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:46.574Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"3de991df-a817-4f5b-aaa1-15390af1eec9","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:46.574Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"3de991df-a817-4f5b-aaa1-15390af1eec9"}
{"level":"info","ts":"2026-06-05T09:43:46.577Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"3de991df-a817-4f5b-aaa1-15390af1eec9","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:43:39Z","endTime":"2026-06-05T09:43:43Z"},"jobId":"counter-dgn8t","rayClusterName":"counter-4qmws","dashboardURL":"counter-4qmws-head-svc.test-ns-t5s4r.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-05T09:43:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:43:31Z","stateTransitionTimes":{"ready":"2026-06-05T09:43:31Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.21","serviceIP":"10.132.0.21","podName":"counter-4qmws-head-gfw44","serviceName":"counter-4qmws-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:43:26Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:43:31Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:43:39Z","endTime":"2026-06-05T09:43:43Z"},"jobId":"counter-dgn8t","rayClusterName":"counter-4qmws","dashboardURL":"counter-4qmws-head-svc.test-ns-t5s4r.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-05T09:43:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:43:31Z","stateTransitionTimes":{"ready":"2026-06-05T09:43:31Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.21","serviceIP":"10.132.0.21","podName":"counter-4qmws-head-gfw44","serviceName":"counter-4qmws-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:43:26Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:43:31Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:43:49.578Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"2eb37b06-98f4-49ec-b43f-cb2dfafb6ee5","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:49.578Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"2eb37b06-98f4-49ec-b43f-cb2dfafb6ee5","name":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:49.578Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"2eb37b06-98f4-49ec-b43f-cb2dfafb6ee5","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:49.578Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"2eb37b06-98f4-49ec-b43f-cb2dfafb6ee5"}
{"level":"info","ts":"2026-06-05T09:43:49.582Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"2eb37b06-98f4-49ec-b43f-cb2dfafb6ee5","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:43:39Z","endTime":"2026-06-05T09:43:43Z"},"jobId":"counter-dgn8t","rayClusterName":"counter-4qmws","dashboardURL":"counter-4qmws-head-svc.test-ns-t5s4r.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-05T09:43:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:43:31Z","stateTransitionTimes":{"ready":"2026-06-05T09:43:31Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.21","serviceIP":"10.132.0.21","podName":"counter-4qmws-head-gfw44","serviceName":"counter-4qmws-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:43:26Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:43:31Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:43:39Z","endTime":"2026-06-05T09:43:43Z"},"jobId":"counter-dgn8t","rayClusterName":"counter-4qmws","dashboardURL":"counter-4qmws-head-svc.test-ns-t5s4r.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-05T09:43:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:43:31Z","stateTransitionTimes":{"ready":"2026-06-05T09:43:31Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.21","serviceIP":"10.132.0.21","podName":"counter-4qmws-head-gfw44","serviceName":"counter-4qmws-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:43:26Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:43:31Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:43:50.503Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"01f946fd-df41-46e5-8549-f97416b5534e","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:50.503Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"01f946fd-df41-46e5-8549-f97416b5534e","name":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:50.503Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"01f946fd-df41-46e5-8549-f97416b5534e","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:50.503Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"01f946fd-df41-46e5-8549-f97416b5534e"}
{"level":"info","ts":"2026-06-05T09:43:50.506Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"01f946fd-df41-46e5-8549-f97416b5534e","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:43:39Z","endTime":"2026-06-05T09:43:43Z"},"jobId":"counter-dgn8t","rayClusterName":"counter-4qmws","dashboardURL":"counter-4qmws-head-svc.test-ns-t5s4r.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-05T09:43:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:43:31Z","stateTransitionTimes":{"ready":"2026-06-05T09:43:31Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.21","serviceIP":"10.132.0.21","podName":"counter-4qmws-head-gfw44","serviceName":"counter-4qmws-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:43:26Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:43:31Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:43:39Z","endTime":"2026-06-05T09:43:43Z"},"jobId":"counter-dgn8t","rayClusterName":"counter-4qmws","dashboardURL":"counter-4qmws-head-svc.test-ns-t5s4r.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-05T09:43:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:43:31Z","stateTransitionTimes":{"ready":"2026-06-05T09:43:31Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.21","serviceIP":"10.132.0.21","podName":"counter-4qmws-head-gfw44","serviceName":"counter-4qmws-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:43:26Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:43:31Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:43:51.512Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"d0794857-c5ed-466d-b136-53a9eb921205","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:51.512Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"d0794857-c5ed-466d-b136-53a9eb921205","name":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:51.512Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"d0794857-c5ed-466d-b136-53a9eb921205","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:51.512Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"d0794857-c5ed-466d-b136-53a9eb921205"}
{"level":"info","ts":"2026-06-05T09:43:51.515Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"d0794857-c5ed-466d-b136-53a9eb921205","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:43:39Z","endTime":"2026-06-05T09:43:43Z"},"jobId":"counter-dgn8t","rayClusterName":"counter-4qmws","dashboardURL":"counter-4qmws-head-svc.test-ns-t5s4r.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-05T09:43:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:43:31Z","stateTransitionTimes":{"ready":"2026-06-05T09:43:31Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.21","serviceIP":"10.132.0.21","podName":"counter-4qmws-head-gfw44","serviceName":"counter-4qmws-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:43:26Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:43:31Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:43:39Z","endTime":"2026-06-05T09:43:43Z"},"jobId":"counter-dgn8t","rayClusterName":"counter-4qmws","dashboardURL":"counter-4qmws-head-svc.test-ns-t5s4r.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-05T09:43:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:43:31Z","stateTransitionTimes":{"ready":"2026-06-05T09:43:31Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.21","serviceIP":"10.132.0.21","podName":"counter-4qmws-head-gfw44","serviceName":"counter-4qmws-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:43:26Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:43:31Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:43:51.532Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"fd76bb7c-d7e1-46d3-8e7a-791b1031242f","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:51.532Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"fd76bb7c-d7e1-46d3-8e7a-791b1031242f","name":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:51.532Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"fd76bb7c-d7e1-46d3-8e7a-791b1031242f","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:51.532Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"fd76bb7c-d7e1-46d3-8e7a-791b1031242f"}
{"level":"info","ts":"2026-06-05T09:43:51.535Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"fd76bb7c-d7e1-46d3-8e7a-791b1031242f","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:43:39Z","endTime":"2026-06-05T09:43:43Z"},"jobId":"counter-dgn8t","rayClusterName":"counter-4qmws","dashboardURL":"counter-4qmws-head-svc.test-ns-t5s4r.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-05T09:43:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:43:31Z","stateTransitionTimes":{"ready":"2026-06-05T09:43:31Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.21","serviceIP":"10.132.0.21","podName":"counter-4qmws-head-gfw44","serviceName":"counter-4qmws-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:43:26Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:43:31Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:43:39Z","endTime":"2026-06-05T09:43:43Z"},"jobId":"counter-dgn8t","rayClusterName":"counter-4qmws","dashboardURL":"counter-4qmws-head-svc.test-ns-t5s4r.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Complete","message":"Job finished successfully.","startTime":"2026-06-05T09:43:03Z","succeeded":1,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:43:31Z","stateTransitionTimes":{"ready":"2026-06-05T09:43:31Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.21","serviceIP":"10.132.0.21","podName":"counter-4qmws-head-gfw44","serviceName":"counter-4qmws-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:43:26Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:43:31Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:03Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:03Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:43:51.535Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"fd76bb7c-d7e1-46d3-8e7a-791b1031242f","old JobStatus":"SUCCEEDED","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Complete"}
{"level":"info","ts":"2026-06-05T09:43:51.552Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"b26d88b4-1b6e-4c2c-8802-5d2f319048fa","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:51.552Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"b26d88b4-1b6e-4c2c-8802-5d2f319048fa","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-05 09:43:51 +0000 UTC","Now":"2026-06-05T09:43:51.552Z","ShutdownTime":"2026-06-05T09:43:51.000Z"}
{"level":"info","ts":"2026-06-05T09:43:51.552Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"b26d88b4-1b6e-4c2c-8802-5d2f319048fa","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-05T09:43:51.562Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob is deleted","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"b26d88b4-1b6e-4c2c-8802-5d2f319048fa","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:51.562Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"b26d88b4-1b6e-4c2c-8802-5d2f319048fa","isClusterDeleted":false}
{"level":"info","ts":"2026-06-05T09:43:51.562Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"b26d88b4-1b6e-4c2c-8802-5d2f319048fa","RayCluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:51.562Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"9fde93b4-b300-4de7-81a8-806a07febf2e","rayCluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:51.563Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"reconcileID":"2cca3bb7-9be4-4326-9cec-8ca367cb5b8b"}
{"level":"info","ts":"2026-06-05T09:43:51.563Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"88b5bf0b-0b9a-4c85-9757-37ef0e90e9ae"}
{"level":"info","ts":"2026-06-05T09:43:51.563Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"24143b70-6246-4eaa-a3cc-ef6079622d63","namespacedName":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:51.563Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"9fde93b4-b300-4de7-81a8-806a07febf2e","pod":"counter-4qmws-head-gfw44","ip":"10.132.0.21"}
{"level":"info","ts":"2026-06-05T09:43:51.563Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"9fde93b4-b300-4de7-81a8-806a07febf2e","pod":"counter-4qmws-small-group-worker-n29x6","ip":"10.133.0.41"}
{"level":"info","ts":"2026-06-05T09:43:51.563Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"24143b70-6246-4eaa-a3cc-ef6079622d63","cluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:51.563Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"24143b70-6246-4eaa-a3cc-ef6079622d63","cluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:51.563Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"9fde93b4-b300-4de7-81a8-806a07febf2e","rayCluster":"counter-4qmws","podIPs":["10.132.0.21","10.133.0.41"]}
{"level":"info","ts":"2026-06-05T09:43:51.563Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"6dbf7f24-7a04-4568-acb0-795b5ae0e79f","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:51.563Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"6dbf7f24-7a04-4568-acb0-795b5ae0e79f","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-05 09:43:51 +0000 UTC","Now":"2026-06-05T09:43:51.563Z","ShutdownTime":"2026-06-05T09:43:51.000Z"}
{"level":"info","ts":"2026-06-05T09:43:51.563Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"6dbf7f24-7a04-4568-acb0-795b5ae0e79f","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-05T09:43:51.563Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"6dbf7f24-7a04-4568-acb0-795b5ae0e79f","RayCluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:51.563Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"6dbf7f24-7a04-4568-acb0-795b5ae0e79f","isClusterDeleted":false}
{"level":"info","ts":"2026-06-05T09:43:51.563Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"6dbf7f24-7a04-4568-acb0-795b5ae0e79f","RayCluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:51.563Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"24143b70-6246-4eaa-a3cc-ef6079622d63","namespace":"test-ns-t5s4r","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-05T09:43:51.568Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"24143b70-6246-4eaa-a3cc-ef6079622d63","name":"kuberay-gateway-access","namespace":"test-ns-t5s4r"}
{"level":"info","ts":"2026-06-05T09:43:51.574Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"24143b70-6246-4eaa-a3cc-ef6079622d63","name":"test-ns-t5s4r-counter-4qmws","namespace":"opendatahub"}
{"level":"info","ts":"2026-06-05T09:43:51.581Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"24143b70-6246-4eaa-a3cc-ef6079622d63","configMap":"kube-rbac-proxy-config-counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:51.586Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"24143b70-6246-4eaa-a3cc-ef6079622d63","serviceAccount":"counter-4qmws-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:43:51.594Z","logger":"raycluster-resource","msg":"default","name":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:51.594Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-4qmws","namespace":"test-ns-t5s4r"}
{"level":"info","ts":"2026-06-05T09:43:51.616Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"24143b70-6246-4eaa-a3cc-ef6079622d63","cluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:51.616Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"7caba9dc-6ad4-482d-acd6-963f8bad8279","namespacedName":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:51.616Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"7caba9dc-6ad4-482d-acd6-963f8bad8279","cluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:51.616Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"7caba9dc-6ad4-482d-acd6-963f8bad8279","cluster":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:51.616Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"7caba9dc-6ad4-482d-acd6-963f8bad8279","namespace":"test-ns-t5s4r","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-05T09:43:51.616Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"d3173b41-ece9-4498-9f07-bcfd97b21778"}
{"level":"info","ts":"2026-06-05T09:43:51.616Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"7caba9dc-6ad4-482d-acd6-963f8bad8279","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-06-05T09:43:51.616Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"1968d8ef-bc16-4b29-968e-191c9b5f1ce5","namespace":"test-ns-t5s4r","clusterName":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:51.617Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"3933030c-d4ee-4d81-9b3e-0c407e4f32f2","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:51.617Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"3933030c-d4ee-4d81-9b3e-0c407e4f32f2","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-05 09:43:51 +0000 UTC","Now":"2026-06-05T09:43:51.617Z","ShutdownTime":"2026-06-05T09:43:51.000Z"}
{"level":"info","ts":"2026-06-05T09:43:51.617Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"3933030c-d4ee-4d81-9b3e-0c407e4f32f2","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-05T09:43:51.617Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"3933030c-d4ee-4d81-9b3e-0c407e4f32f2","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:51.617Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"3933030c-d4ee-4d81-9b3e-0c407e4f32f2","isClusterDeleted":true}
{"level":"info","ts":"2026-06-05T09:43:51.617Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"3933030c-d4ee-4d81-9b3e-0c407e4f32f2","RayCluster":"counter-4qmws"}
{"level":"error","ts":"2026-06-05T09:43:51.620Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"7caba9dc-6ad4-482d-acd6-963f8bad8279","error":"rayclusters.ray.io \"counter-4qmws\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:214\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:155\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:51.621Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"7caba9dc-6ad4-482d-acd6-963f8bad8279"}
{"level":"error","ts":"2026-06-05T09:43:51.621Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"7caba9dc-6ad4-482d-acd6-963f8bad8279","error":"rayclusters.ray.io \"counter-4qmws\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:51.621Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"32ef77bf-e6ae-4cfb-b8b2-b3ed633046ca","namespacedName":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:51.621Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"32ef77bf-e6ae-4cfb-b8b2-b3ed633046ca"}
{"level":"info","ts":"2026-06-05T09:43:51.621Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"32ef77bf-e6ae-4cfb-b8b2-b3ed633046ca","namespace":"test-ns-t5s4r","authClusters":0}
{"level":"info","ts":"2026-06-05T09:43:51.621Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"32ef77bf-e6ae-4cfb-b8b2-b3ed633046ca","namespace":"test-ns-t5s4r"}
{"level":"info","ts":"2026-06-05T09:43:51.626Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"c35799e6-8bff-4901-842e-184844ec1166","namespacedName":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:51.626Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"c35799e6-8bff-4901-842e-184844ec1166"}
{"level":"info","ts":"2026-06-05T09:43:51.626Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"c35799e6-8bff-4901-842e-184844ec1166","namespace":"test-ns-t5s4r","authClusters":0}
{"level":"info","ts":"2026-06-05T09:43:51.626Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"c35799e6-8bff-4901-842e-184844ec1166","namespace":"test-ns-t5s4r"}
{"level":"info","ts":"2026-06-05T09:43:51.634Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"7a40b2ac-f194-45b6-bb25-50e8765a7610"}
{"level":"info","ts":"2026-06-05T09:43:51.638Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"98dae57d-d4a0-473c-9634-fd5f4fd5903f"}
{"level":"info","ts":"2026-06-05T09:43:51.638Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"26df9ac1-3fe4-484b-971f-79d47f8bdad8","namespacedName":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"}}
{"level":"info","ts":"2026-06-05T09:43:51.638Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"26df9ac1-3fe4-484b-971f-79d47f8bdad8"}
{"level":"info","ts":"2026-06-05T09:43:51.638Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"26df9ac1-3fe4-484b-971f-79d47f8bdad8","namespace":"test-ns-t5s4r","authClusters":0}
{"level":"info","ts":"2026-06-05T09:43:51.638Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"26df9ac1-3fe4-484b-971f-79d47f8bdad8","namespace":"test-ns-t5s4r"}
{"level":"info","ts":"2026-06-05T09:43:51.706Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"3f670469-d5c7-4bd2-8c5f-65c7ebc14ce6","DeletionTimestamp":"2026-06-05 09:43:51 +0000 UTC"}
{"level":"info","ts":"2026-06-05T09:43:51.706Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"3f670469-d5c7-4bd2-8c5f-65c7ebc14ce6","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-05T09:43:51.712Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"1968d8ef-bc16-4b29-968e-191c9b5f1ce5","namespace":"test-ns-t5s4r","clusterName":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:43:51.721Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"883185b1-53db-41a3-82ce-1c3784138295"}
{"level":"info","ts":"2026-06-05T09:43:51.728Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"62a64907-e858-437d-a9f4-654e66215fb6"}
{"level":"info","ts":"2026-06-05T09:43:52.582Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"counter","namespace":"test-ns-t5s4r"},"reconcileID":"88a9dd24-895e-4bcd-a34f-ca2c6788c8ce"}
{"level":"info","ts":"2026-06-05T09:43:54.845Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"d8afb4ab-1a48-480a-b42b-e7b40cd8b5a0","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:54.845Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"d8afb4ab-1a48-480a-b42b-e7b40cd8b5a0","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-05T09:43:54.859Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"d8afb4ab-1a48-480a-b42b-e7b40cd8b5a0"}
{"level":"info","ts":"2026-06-05T09:43:54.859Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"d8afb4ab-1a48-480a-b42b-e7b40cd8b5a0","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-05T09:43:54.859Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"d8afb4ab-1a48-480a-b42b-e7b40cd8b5a0","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-np762","rayClusterName":"counter-nl7mr","jobDeploymentStatus":"Initializing","startTime":"2026-06-05T09:43:54Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:43:54.859Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"d8afb4ab-1a48-480a-b42b-e7b40cd8b5a0","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-05T09:43:54.872Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"a5610dc1-23d4-4154-b6df-a998ec776418","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:54.872Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"a5610dc1-23d4-4154-b6df-a998ec776418"}
{"level":"info","ts":"2026-06-05T09:43:54.872Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"a5610dc1-23d4-4154-b6df-a998ec776418","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-05T09:43:54.872Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"a5610dc1-23d4-4154-b6df-a998ec776418","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-qxkcl","rayClusterName":"counter-xqwwk","jobDeploymentStatus":"Initializing","startTime":"2026-06-05T09:43:54Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:43:54.872Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"a5610dc1-23d4-4154-b6df-a998ec776418","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-05T09:43:54.878Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"a5610dc1-23d4-4154-b6df-a998ec776418","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-06-05T09:43:54.878Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"a5610dc1-23d4-4154-b6df-a998ec776418"}
{"level":"error","ts":"2026-06-05T09:43:54.878Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"a5610dc1-23d4-4154-b6df-a998ec776418","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:54.878Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"063edf07-3fed-471d-bfa4-940021c6e5b8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:54.878Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"063edf07-3fed-471d-bfa4-940021c6e5b8","name":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:43:54.878Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"063edf07-3fed-471d-bfa4-940021c6e5b8","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:43:54.878Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"063edf07-3fed-471d-bfa4-940021c6e5b8","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:43:54.883Z","logger":"raycluster-resource","msg":"default","name":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:43:54.883Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-nl7mr","namespace":"test-ns-thrrw"}
{"level":"info","ts":"2026-06-05T09:43:54.898Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"063edf07-3fed-471d-bfa4-940021c6e5b8","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:43:54.898Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"fff7ee4e-0a7f-4961-84db-f890eb01a23c","namespacedName":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:43:54.898Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"3b21bef0-df69-4d70-a5bc-0e792f3d27ef","rayCluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:43:54.898Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"a8bed543-29c8-4ad7-9f7b-2d89e6180b2a"}
{"level":"info","ts":"2026-06-05T09:43:54.898Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"063edf07-3fed-471d-bfa4-940021c6e5b8"}
{"level":"info","ts":"2026-06-05T09:43:54.898Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"3b21bef0-df69-4d70-a5bc-0e792f3d27ef","rayCluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:43:54.898Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"063edf07-3fed-471d-bfa4-940021c6e5b8","RayCluster":"counter-nl7mr","State":""}
{"level":"info","ts":"2026-06-05T09:43:54.898Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"fff7ee4e-0a7f-4961-84db-f890eb01a23c","mode":"IntegratedOAuth","cluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:43:54.898Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"fff7ee4e-0a7f-4961-84db-f890eb01a23c","cluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:43:54.898Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"a8bed543-29c8-4ad7-9f7b-2d89e6180b2a"}
{"level":"info","ts":"2026-06-05T09:43:54.898Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"38544c3b-67b1-42d3-abc7-6b86a9467c0f","cluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:43:54.898Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"4d993201-bcef-4a4a-8b2e-07c22b6f6f40","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:54.898Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"4d993201-bcef-4a4a-8b2e-07c22b6f6f40","name":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:43:54.898Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"fff7ee4e-0a7f-4961-84db-f890eb01a23c","cluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:43:54.898Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"4d993201-bcef-4a4a-8b2e-07c22b6f6f40","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:43:54.898Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"4d993201-bcef-4a4a-8b2e-07c22b6f6f40"}
{"level":"info","ts":"2026-06-05T09:43:54.898Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"4d993201-bcef-4a4a-8b2e-07c22b6f6f40","RayCluster":"counter-nl7mr","State":""}
{"level":"info","ts":"2026-06-05T09:43:54.904Z","logger":"raycluster-resource","msg":"default","name":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:43:54.904Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-nl7mr","namespace":"test-ns-thrrw"}
{"level":"info","ts":"2026-06-05T09:43:54.906Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"a8bed543-29c8-4ad7-9f7b-2d89e6180b2a","name":"counter-nl7mr-head-svc"}
{"level":"info","ts":"2026-06-05T09:43:54.906Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"a8bed543-29c8-4ad7-9f7b-2d89e6180b2a","error":"head certificate secret ray-head-secret-counter-nl7mr not found"}
{"level":"error","ts":"2026-06-05T09:43:54.907Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"a8bed543-29c8-4ad7-9f7b-2d89e6180b2a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-nl7mr not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:54.907Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"a8bed543-29c8-4ad7-9f7b-2d89e6180b2a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-nl7mr not found"}
{"level":"info","ts":"2026-06-05T09:43:54.907Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"a8bed543-29c8-4ad7-9f7b-2d89e6180b2a","filter labels":["test-ns-thrrw",{"ray.io/cluster":"counter-nl7mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:54.907Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"a8bed543-29c8-4ad7-9f7b-2d89e6180b2a","filter labels":["test-ns-thrrw",{"ray.io/cluster":"counter-nl7mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:54.907Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"a8bed543-29c8-4ad7-9f7b-2d89e6180b2a","filter labels":["test-ns-thrrw",{"ray.io/cluster":"counter-nl7mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:54.907Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"a8bed543-29c8-4ad7-9f7b-2d89e6180b2a","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-05T09:43:54.907Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"a8bed543-29c8-4ad7-9f7b-2d89e6180b2a","name":"counter-nl7mr","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:43:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-nl7mr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:43:54.912Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"3b21bef0-df69-4d70-a5bc-0e792f3d27ef","rayCluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:43:54.912Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"3b21bef0-df69-4d70-a5bc-0e792f3d27ef","rayCluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:43:54.914Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"38544c3b-67b1-42d3-abc7-6b86a9467c0f","cluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:43:54.915Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"cdef4ba8-eb67-496f-8efb-656a01a30e29","cluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:43:54.916Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"db6ea266-a79a-4835-b33d-0110f47fc86a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:54.916Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"db6ea266-a79a-4835-b33d-0110f47fc86a","name":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:43:54.916Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"db6ea266-a79a-4835-b33d-0110f47fc86a","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:43:54.916Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"db6ea266-a79a-4835-b33d-0110f47fc86a"}
{"level":"info","ts":"2026-06-05T09:43:54.916Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"db6ea266-a79a-4835-b33d-0110f47fc86a","RayCluster":"counter-nl7mr","State":""}
{"level":"info","ts":"2026-06-05T09:43:54.919Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"a8bed543-29c8-4ad7-9f7b-2d89e6180b2a","name":"counter-nl7mr","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-nl7mr\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-thrrw","name":"counter-nl7mr"}}
{"level":"info","ts":"2026-06-05T09:43:54.919Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"a8bed543-29c8-4ad7-9f7b-2d89e6180b2a"}
{"level":"error","ts":"2026-06-05T09:43:54.919Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"a8bed543-29c8-4ad7-9f7b-2d89e6180b2a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-nl7mr not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:54.919Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"377219c8-7326-4875-9e42-3fa429ccc8ca"}
{"level":"info","ts":"2026-06-05T09:43:54.919Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"377219c8-7326-4875-9e42-3fa429ccc8ca"}
{"level":"info","ts":"2026-06-05T09:43:54.919Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"377219c8-7326-4875-9e42-3fa429ccc8ca","1 head service found":"counter-nl7mr-head-svc"}
{"level":"info","ts":"2026-06-05T09:43:54.919Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"377219c8-7326-4875-9e42-3fa429ccc8ca","error":"head certificate secret ray-head-secret-counter-nl7mr not found"}
{"level":"error","ts":"2026-06-05T09:43:54.919Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"377219c8-7326-4875-9e42-3fa429ccc8ca","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-nl7mr not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:54.919Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"377219c8-7326-4875-9e42-3fa429ccc8ca","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-nl7mr not found"}
{"level":"info","ts":"2026-06-05T09:43:54.919Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"377219c8-7326-4875-9e42-3fa429ccc8ca","filter labels":["test-ns-thrrw",{"ray.io/cluster":"counter-nl7mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:54.919Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"377219c8-7326-4875-9e42-3fa429ccc8ca","filter labels":["test-ns-thrrw",{"ray.io/cluster":"counter-nl7mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:54.919Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"377219c8-7326-4875-9e42-3fa429ccc8ca","filter labels":["test-ns-thrrw",{"ray.io/cluster":"counter-nl7mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:54.919Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"377219c8-7326-4875-9e42-3fa429ccc8ca","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-05T09:43:54.919Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"377219c8-7326-4875-9e42-3fa429ccc8ca","name":"counter-nl7mr","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:43:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-nl7mr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:43:54.923Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"fff7ee4e-0a7f-4961-84db-f890eb01a23c","name":"counter-nl7mr-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:43:55.006Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"3b21bef0-df69-4d70-a5bc-0e792f3d27ef"}
{"level":"info","ts":"2026-06-05T09:43:55.006Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"cdef4ba8-eb67-496f-8efb-656a01a30e29","name":"counter-nl7mr-head"}
{"level":"info","ts":"2026-06-05T09:43:55.006Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"3b21bef0-df69-4d70-a5bc-0e792f3d27ef","rayCluster":"counter-nl7mr","secret":"ca-secret"}
{"level":"info","ts":"2026-06-05T09:43:55.006Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"3b21bef0-df69-4d70-a5bc-0e792f3d27ef","rayCluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:43:55.006Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"fff7ee4e-0a7f-4961-84db-f890eb01a23c","name":"kuberay-gateway-access","namespace":"test-ns-thrrw","operation":"created"}
{"level":"info","ts":"2026-06-05T09:43:55.007Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"377219c8-7326-4875-9e42-3fa429ccc8ca"}
{"level":"error","ts":"2026-06-05T09:43:55.007Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"377219c8-7326-4875-9e42-3fa429ccc8ca","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-nl7mr not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:55.007Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"5c71edb9-8491-45e4-9a21-8b5bf3a53fc1"}
{"level":"info","ts":"2026-06-05T09:43:55.007Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"5c71edb9-8491-45e4-9a21-8b5bf3a53fc1"}
{"level":"info","ts":"2026-06-05T09:43:55.007Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"5c71edb9-8491-45e4-9a21-8b5bf3a53fc1","1 head service found":"counter-nl7mr-head-svc"}
{"level":"info","ts":"2026-06-05T09:43:55.007Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"5c71edb9-8491-45e4-9a21-8b5bf3a53fc1","error":"head certificate secret ray-head-secret-counter-nl7mr not found"}
{"level":"info","ts":"2026-06-05T09:43:55.007Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"3bd1b294-90df-4cf1-9561-d01b771c7499","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"error","ts":"2026-06-05T09:43:55.007Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"5c71edb9-8491-45e4-9a21-8b5bf3a53fc1","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-nl7mr not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:55.007Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"3bd1b294-90df-4cf1-9561-d01b771c7499","name":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:43:55.007Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"5c71edb9-8491-45e4-9a21-8b5bf3a53fc1","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-nl7mr not found"}
{"level":"info","ts":"2026-06-05T09:43:55.007Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"3bd1b294-90df-4cf1-9561-d01b771c7499","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:43:55.007Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"5c71edb9-8491-45e4-9a21-8b5bf3a53fc1","filter labels":["test-ns-thrrw",{"ray.io/cluster":"counter-nl7mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:55.007Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"5c71edb9-8491-45e4-9a21-8b5bf3a53fc1","filter labels":["test-ns-thrrw",{"ray.io/cluster":"counter-nl7mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:55.007Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"5c71edb9-8491-45e4-9a21-8b5bf3a53fc1","filter labels":["test-ns-thrrw",{"ray.io/cluster":"counter-nl7mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:55.007Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"5c71edb9-8491-45e4-9a21-8b5bf3a53fc1"}
{"level":"info","ts":"2026-06-05T09:43:55.007Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"3bd1b294-90df-4cf1-9561-d01b771c7499"}
{"level":"error","ts":"2026-06-05T09:43:55.007Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"5c71edb9-8491-45e4-9a21-8b5bf3a53fc1","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-nl7mr not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:55.007Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"3bd1b294-90df-4cf1-9561-d01b771c7499","RayCluster":"counter-nl7mr","State":""}
{"level":"info","ts":"2026-06-05T09:43:55.015Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"fff7ee4e-0a7f-4961-84db-f890eb01a23c","name":"test-ns-thrrw-counter-nl7mr","operation":"created"}
{"level":"info","ts":"2026-06-05T09:43:55.017Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"4f006c1f-9a20-4bb6-8767-01445c8dea7f"}
{"level":"info","ts":"2026-06-05T09:43:55.017Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"4f006c1f-9a20-4bb6-8767-01445c8dea7f"}
{"level":"info","ts":"2026-06-05T09:43:55.017Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"4f006c1f-9a20-4bb6-8767-01445c8dea7f","1 head service found":"counter-nl7mr-head-svc"}
{"level":"info","ts":"2026-06-05T09:43:55.017Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"4f006c1f-9a20-4bb6-8767-01445c8dea7f","error":"head certificate secret ray-head-secret-counter-nl7mr not found"}
{"level":"error","ts":"2026-06-05T09:43:55.017Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"4f006c1f-9a20-4bb6-8767-01445c8dea7f","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-nl7mr not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:55.018Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"4f006c1f-9a20-4bb6-8767-01445c8dea7f","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-nl7mr not found"}
{"level":"info","ts":"2026-06-05T09:43:55.018Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"4f006c1f-9a20-4bb6-8767-01445c8dea7f","filter labels":["test-ns-thrrw",{"ray.io/cluster":"counter-nl7mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:55.018Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"4f006c1f-9a20-4bb6-8767-01445c8dea7f","filter labels":["test-ns-thrrw",{"ray.io/cluster":"counter-nl7mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:55.018Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"4f006c1f-9a20-4bb6-8767-01445c8dea7f","filter labels":["test-ns-thrrw",{"ray.io/cluster":"counter-nl7mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:55.018Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"4f006c1f-9a20-4bb6-8767-01445c8dea7f"}
{"level":"error","ts":"2026-06-05T09:43:55.018Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"4f006c1f-9a20-4bb6-8767-01445c8dea7f","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-nl7mr not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:55.018Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"3b21bef0-df69-4d70-a5bc-0e792f3d27ef","rayCluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:43:55.021Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"fff7ee4e-0a7f-4961-84db-f890eb01a23c","name":"kube-rbac-proxy-config-counter-nl7mr","operation":"created"}
{"level":"info","ts":"2026-06-05T09:43:55.022Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"cdef4ba8-eb67-496f-8efb-656a01a30e29","name":"counter-nl7mr-workers"}
{"level":"info","ts":"2026-06-05T09:43:55.022Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"cdef4ba8-eb67-496f-8efb-656a01a30e29","cluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:43:55.022Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"1ff29a54-3c36-4f3b-ae9e-b6aa627978ff","cluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:43:55.030Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"3b21bef0-df69-4d70-a5bc-0e792f3d27ef"}
{"level":"info","ts":"2026-06-05T09:43:55.030Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"3b21bef0-df69-4d70-a5bc-0e792f3d27ef","rayCluster":"counter-nl7mr","podIPs":[]}
{"level":"info","ts":"2026-06-05T09:43:55.105Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"91dbade3-2091-459b-8077-afbb91575849"}
{"level":"info","ts":"2026-06-05T09:43:55.105Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"91dbade3-2091-459b-8077-afbb91575849"}
{"level":"info","ts":"2026-06-05T09:43:55.105Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"3b21bef0-df69-4d70-a5bc-0e792f3d27ef"}
{"level":"info","ts":"2026-06-05T09:43:55.106Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"91dbade3-2091-459b-8077-afbb91575849","1 head service found":"counter-nl7mr-head-svc"}
{"level":"info","ts":"2026-06-05T09:43:55.106Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"91dbade3-2091-459b-8077-afbb91575849","error":"head certificate secret ray-head-secret-counter-nl7mr not found"}
{"level":"error","ts":"2026-06-05T09:43:55.106Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"fff7ee4e-0a7f-4961-84db-f890eb01a23c","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-nl7mr\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-05T09:43:55.106Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"91dbade3-2091-459b-8077-afbb91575849","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-nl7mr not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:55.106Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"91dbade3-2091-459b-8077-afbb91575849","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-nl7mr not found"}
{"level":"info","ts":"2026-06-05T09:43:55.106Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"fff7ee4e-0a7f-4961-84db-f890eb01a23c"}
{"level":"error","ts":"2026-06-05T09:43:55.106Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"fff7ee4e-0a7f-4961-84db-f890eb01a23c","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-nl7mr\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:55.106Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"91dbade3-2091-459b-8077-afbb91575849","filter labels":["test-ns-thrrw",{"ray.io/cluster":"counter-nl7mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:55.106Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"7b4aa5ff-67af-4f37-bad7-16d7028d07a7","namespacedName":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:43:55.106Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"3b21bef0-df69-4d70-a5bc-0e792f3d27ef","rayCluster":"counter-nl7mr","podIPs":[]}
{"level":"info","ts":"2026-06-05T09:43:55.106Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"91dbade3-2091-459b-8077-afbb91575849","filter labels":["test-ns-thrrw",{"ray.io/cluster":"counter-nl7mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:55.106Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"91dbade3-2091-459b-8077-afbb91575849","filter labels":["test-ns-thrrw",{"ray.io/cluster":"counter-nl7mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:55.106Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"91dbade3-2091-459b-8077-afbb91575849"}
{"level":"error","ts":"2026-06-05T09:43:55.106Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"91dbade3-2091-459b-8077-afbb91575849","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-nl7mr not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:55.106Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"7b4aa5ff-67af-4f37-bad7-16d7028d07a7","mode":"IntegratedOAuth","cluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:43:55.106Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"7b4aa5ff-67af-4f37-bad7-16d7028d07a7","cluster":"counter-nl7mr"}
{"level":"error","ts":"2026-06-05T09:43:55.106Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"3b21bef0-df69-4d70-a5bc-0e792f3d27ef","error":"Certificate.cert-manager.io \"ray-worker-cert-counter-nl7mr\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:55.106Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"3b21bef0-df69-4d70-a5bc-0e792f3d27ef"}
{"level":"error","ts":"2026-06-05T09:43:55.106Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"3b21bef0-df69-4d70-a5bc-0e792f3d27ef","error":"Certificate.cert-manager.io \"ray-worker-cert-counter-nl7mr\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:55.106Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"02d1f1f5-ee9f-444a-ba9c-b6f37651f93e","rayCluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:43:55.106Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"02d1f1f5-ee9f-444a-ba9c-b6f37651f93e"}
{"level":"info","ts":"2026-06-05T09:43:55.111Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"1ff29a54-3c36-4f3b-ae9e-b6aa627978ff","name":"counter-nl7mr-head"}
{"level":"info","ts":"2026-06-05T09:43:55.112Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"86b1bc66-aa18-41b4-b05b-53a876685174","rayCluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:43:55.112Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"86b1bc66-aa18-41b4-b05b-53a876685174"}
{"level":"info","ts":"2026-06-05T09:43:55.119Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"7b4aa5ff-67af-4f37-bad7-16d7028d07a7","cluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:43:55.119Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"0fcebc54-15fe-442a-927b-0cb5b7f37381","namespacedName":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:43:55.120Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"0fcebc54-15fe-442a-927b-0cb5b7f37381","mode":"IntegratedOAuth","cluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:43:55.120Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"0fcebc54-15fe-442a-927b-0cb5b7f37381","cluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:43:55.121Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"5a6a8e28-bc03-4b7a-88a4-8ec5f7b30c08","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:55.121Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"5a6a8e28-bc03-4b7a-88a4-8ec5f7b30c08","name":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:43:55.121Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"5a6a8e28-bc03-4b7a-88a4-8ec5f7b30c08","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:43:55.121Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"5a6a8e28-bc03-4b7a-88a4-8ec5f7b30c08"}
{"level":"info","ts":"2026-06-05T09:43:55.121Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"5a6a8e28-bc03-4b7a-88a4-8ec5f7b30c08","RayCluster":"counter-nl7mr","State":""}
{"level":"info","ts":"2026-06-05T09:43:55.121Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"02d860c5-ec54-4c43-a0d9-27a1f6dbec2b","rayCluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:43:55.121Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"02d860c5-ec54-4c43-a0d9-27a1f6dbec2b"}
{"level":"info","ts":"2026-06-05T09:43:55.126Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"1ff29a54-3c36-4f3b-ae9e-b6aa627978ff","name":"counter-nl7mr-workers"}
{"level":"info","ts":"2026-06-05T09:43:55.126Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"1ff29a54-3c36-4f3b-ae9e-b6aa627978ff","cluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:43:55.126Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"ac15c2bd-e4f2-43a5-b096-d60af7315436","cluster":"counter-nl7mr"}
{"level":"error","ts":"2026-06-05T09:43:55.128Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"0fcebc54-15fe-442a-927b-0cb5b7f37381","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-nl7mr\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:55.128Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"0fcebc54-15fe-442a-927b-0cb5b7f37381"}
{"level":"error","ts":"2026-06-05T09:43:55.128Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"0fcebc54-15fe-442a-927b-0cb5b7f37381","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-nl7mr\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:55.205Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"2e051bb1-7639-4773-9fb5-c9c2c85ffdb4","namespacedName":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:43:55.205Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"2e051bb1-7639-4773-9fb5-c9c2c85ffdb4","mode":"IntegratedOAuth","cluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:43:55.205Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"2e051bb1-7639-4773-9fb5-c9c2c85ffdb4","cluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:43:55.205Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"23923f91-99a7-4624-a38e-e0499aca6f6a"}
{"level":"info","ts":"2026-06-05T09:43:55.205Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"23923f91-99a7-4624-a38e-e0499aca6f6a"}
{"level":"info","ts":"2026-06-05T09:43:55.206Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"23923f91-99a7-4624-a38e-e0499aca6f6a","1 head service found":"counter-nl7mr-head-svc"}
{"level":"info","ts":"2026-06-05T09:43:55.206Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"23923f91-99a7-4624-a38e-e0499aca6f6a","error":"head certificate secret ray-head-secret-counter-nl7mr not found"}
{"level":"error","ts":"2026-06-05T09:43:55.206Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"23923f91-99a7-4624-a38e-e0499aca6f6a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-nl7mr not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:55.206Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"23923f91-99a7-4624-a38e-e0499aca6f6a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-nl7mr not found"}
{"level":"info","ts":"2026-06-05T09:43:55.206Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"23923f91-99a7-4624-a38e-e0499aca6f6a","filter labels":["test-ns-thrrw",{"ray.io/cluster":"counter-nl7mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:55.206Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"23923f91-99a7-4624-a38e-e0499aca6f6a","filter labels":["test-ns-thrrw",{"ray.io/cluster":"counter-nl7mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:55.206Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"23923f91-99a7-4624-a38e-e0499aca6f6a","filter labels":["test-ns-thrrw",{"ray.io/cluster":"counter-nl7mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:55.206Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"23923f91-99a7-4624-a38e-e0499aca6f6a"}
{"level":"error","ts":"2026-06-05T09:43:55.206Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"23923f91-99a7-4624-a38e-e0499aca6f6a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-nl7mr not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:55.210Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"ac15c2bd-e4f2-43a5-b096-d60af7315436","name":"counter-nl7mr-head"}
{"level":"info","ts":"2026-06-05T09:43:55.217Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"2e051bb1-7639-4773-9fb5-c9c2c85ffdb4","cluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:43:55.220Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"ac15c2bd-e4f2-43a5-b096-d60af7315436","name":"counter-nl7mr-workers"}
{"level":"info","ts":"2026-06-05T09:43:55.220Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"ac15c2bd-e4f2-43a5-b096-d60af7315436","cluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:43:55.366Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"14dd3be2-a22a-4edb-9cf0-a8da37e2f7ba"}
{"level":"info","ts":"2026-06-05T09:43:55.366Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"14dd3be2-a22a-4edb-9cf0-a8da37e2f7ba"}
{"level":"info","ts":"2026-06-05T09:43:55.366Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"14dd3be2-a22a-4edb-9cf0-a8da37e2f7ba","1 head service found":"counter-nl7mr-head-svc"}
{"level":"info","ts":"2026-06-05T09:43:55.366Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"14dd3be2-a22a-4edb-9cf0-a8da37e2f7ba","error":"head certificate secret ray-head-secret-counter-nl7mr not found"}
{"level":"error","ts":"2026-06-05T09:43:55.366Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"14dd3be2-a22a-4edb-9cf0-a8da37e2f7ba","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-nl7mr not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:55.366Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"14dd3be2-a22a-4edb-9cf0-a8da37e2f7ba","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-nl7mr not found"}
{"level":"info","ts":"2026-06-05T09:43:55.366Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"14dd3be2-a22a-4edb-9cf0-a8da37e2f7ba","filter labels":["test-ns-thrrw",{"ray.io/cluster":"counter-nl7mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:55.366Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"14dd3be2-a22a-4edb-9cf0-a8da37e2f7ba","filter labels":["test-ns-thrrw",{"ray.io/cluster":"counter-nl7mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:55.366Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"14dd3be2-a22a-4edb-9cf0-a8da37e2f7ba","filter labels":["test-ns-thrrw",{"ray.io/cluster":"counter-nl7mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:55.366Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"14dd3be2-a22a-4edb-9cf0-a8da37e2f7ba"}
{"level":"error","ts":"2026-06-05T09:43:55.366Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"14dd3be2-a22a-4edb-9cf0-a8da37e2f7ba","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-nl7mr not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:55.687Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"0474fc95-d99d-4c7b-b57e-43f9b7f13b39"}
{"level":"info","ts":"2026-06-05T09:43:55.687Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"0474fc95-d99d-4c7b-b57e-43f9b7f13b39"}
{"level":"info","ts":"2026-06-05T09:43:55.687Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"0474fc95-d99d-4c7b-b57e-43f9b7f13b39","1 head service found":"counter-nl7mr-head-svc"}
{"level":"info","ts":"2026-06-05T09:43:55.687Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"0474fc95-d99d-4c7b-b57e-43f9b7f13b39","error":"head certificate secret ray-head-secret-counter-nl7mr not found"}
{"level":"error","ts":"2026-06-05T09:43:55.687Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"0474fc95-d99d-4c7b-b57e-43f9b7f13b39","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-nl7mr not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:55.687Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"0474fc95-d99d-4c7b-b57e-43f9b7f13b39","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-nl7mr not found"}
{"level":"info","ts":"2026-06-05T09:43:55.687Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"0474fc95-d99d-4c7b-b57e-43f9b7f13b39","filter labels":["test-ns-thrrw",{"ray.io/cluster":"counter-nl7mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:55.687Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"0474fc95-d99d-4c7b-b57e-43f9b7f13b39","filter labels":["test-ns-thrrw",{"ray.io/cluster":"counter-nl7mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:55.687Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"0474fc95-d99d-4c7b-b57e-43f9b7f13b39","filter labels":["test-ns-thrrw",{"ray.io/cluster":"counter-nl7mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:55.687Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"0474fc95-d99d-4c7b-b57e-43f9b7f13b39"}
{"level":"error","ts":"2026-06-05T09:43:55.687Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"0474fc95-d99d-4c7b-b57e-43f9b7f13b39","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-nl7mr not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:56.327Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"75f60b57-d55e-4045-b47e-9580b386c243"}
{"level":"info","ts":"2026-06-05T09:43:56.327Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"75f60b57-d55e-4045-b47e-9580b386c243"}
{"level":"info","ts":"2026-06-05T09:43:56.327Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"75f60b57-d55e-4045-b47e-9580b386c243","1 head service found":"counter-nl7mr-head-svc"}
{"level":"info","ts":"2026-06-05T09:43:56.327Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"75f60b57-d55e-4045-b47e-9580b386c243","error":"head certificate secret ray-head-secret-counter-nl7mr not found"}
{"level":"error","ts":"2026-06-05T09:43:56.327Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"75f60b57-d55e-4045-b47e-9580b386c243","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-nl7mr not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:56.328Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"75f60b57-d55e-4045-b47e-9580b386c243","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-nl7mr not found"}
{"level":"info","ts":"2026-06-05T09:43:56.328Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"75f60b57-d55e-4045-b47e-9580b386c243","filter labels":["test-ns-thrrw",{"ray.io/cluster":"counter-nl7mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:56.328Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"75f60b57-d55e-4045-b47e-9580b386c243","filter labels":["test-ns-thrrw",{"ray.io/cluster":"counter-nl7mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:56.328Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"75f60b57-d55e-4045-b47e-9580b386c243","filter labels":["test-ns-thrrw",{"ray.io/cluster":"counter-nl7mr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:43:56.328Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"75f60b57-d55e-4045-b47e-9580b386c243"}
{"level":"error","ts":"2026-06-05T09:43:56.328Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"75f60b57-d55e-4045-b47e-9580b386c243","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-nl7mr not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:57.608Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"473793e5-fe3e-4685-b036-426678951169"}
{"level":"info","ts":"2026-06-05T09:43:57.608Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"473793e5-fe3e-4685-b036-426678951169"}
{"level":"info","ts":"2026-06-05T09:43:57.608Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"473793e5-fe3e-4685-b036-426678951169","1 head service found":"counter-nl7mr-head-svc"}
{"level":"info","ts":"2026-06-05T09:43:57.608Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"473793e5-fe3e-4685-b036-426678951169"}
{"level":"info","ts":"2026-06-05T09:43:57.608Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"473793e5-fe3e-4685-b036-426678951169","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:43:57.608Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"473793e5-fe3e-4685-b036-426678951169"}
{"level":"info","ts":"2026-06-05T09:43:57.608Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"473793e5-fe3e-4685-b036-426678951169","mode":"IntegratedOAuth","cluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:43:57.608Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"473793e5-fe3e-4685-b036-426678951169","cluster":"counter-nl7mr","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:43:57.608Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"473793e5-fe3e-4685-b036-426678951169","cluster":"counter-nl7mr","authType":"IntegratedOAuth","serviceAccount":"counter-nl7mr-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-05T09:43:57.608Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"473793e5-fe3e-4685-b036-426678951169","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"counter-nl7mr","ray.io/group":"headgroup","ray.io/identifier":"counter-nl7mr-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-05T09:43:57.608Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"473793e5-fe3e-4685-b036-426678951169","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"counter-nl7mr-head-svc.test-ns-thrrw.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-05T09:43:57.609Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"473793e5-fe3e-4685-b036-426678951169","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=counter-nl7mr-head-svc.test-ns-thrrw.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-05T09:43:57.609Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"473793e5-fe3e-4685-b036-426678951169","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=counter-nl7mr-head-svc.test-ns-thrrw.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-05T09:43:57.609Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"473793e5-fe3e-4685-b036-426678951169","enabled":true}
{"level":"info","ts":"2026-06-05T09:43:57.628Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"473793e5-fe3e-4685-b036-426678951169","name":"counter-nl7mr-head-7xdm5"}
{"level":"info","ts":"2026-06-05T09:43:57.628Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"473793e5-fe3e-4685-b036-426678951169","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:43:57.628Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"473793e5-fe3e-4685-b036-426678951169","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:57.628Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"473793e5-fe3e-4685-b036-426678951169","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-05T09:43:57.628Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"473793e5-fe3e-4685-b036-426678951169","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:57.628Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"473793e5-fe3e-4685-b036-426678951169","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-05T09:43:57.628Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"473793e5-fe3e-4685-b036-426678951169"}
{"level":"info","ts":"2026-06-05T09:43:57.628Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"473793e5-fe3e-4685-b036-426678951169","nodeType":"worker","rayStartParams":{"address":"counter-nl7mr-head-svc.test-ns-thrrw.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-05T09:43:57.628Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"473793e5-fe3e-4685-b036-426678951169","rayStartCmd":"ray start  --address=counter-nl7mr-head-svc.test-ns-thrrw.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:43:57.628Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"473793e5-fe3e-4685-b036-426678951169","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=counter-nl7mr-head-svc.test-ns-thrrw.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:43:57.628Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"473793e5-fe3e-4685-b036-426678951169","enabled":true}
{"level":"info","ts":"2026-06-05T09:43:57.642Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"473793e5-fe3e-4685-b036-426678951169","name":"counter-nl7mr-small-group-worker-wpkdn"}
{"level":"info","ts":"2026-06-05T09:43:57.642Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"473793e5-fe3e-4685-b036-426678951169","podName":"counter-nl7mr-head-7xdm5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:43:57.642Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"473793e5-fe3e-4685-b036-426678951169","podName":"counter-nl7mr-head-7xdm5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:43:57.642Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"473793e5-fe3e-4685-b036-426678951169","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"counter-nl7mr-head-svc"},"newHead":{"podName":"counter-nl7mr-head-7xdm5","serviceName":"counter-nl7mr-head-svc"}}
{"level":"info","ts":"2026-06-05T09:43:57.642Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"473793e5-fe3e-4685-b036-426678951169","name":"counter-nl7mr","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:43:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-nl7mr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:55Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:43:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-nl7mr-head-7xdm5","serviceName":"counter-nl7mr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:55Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:43:57.707Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"7532ae9c-8762-4573-88d9-b0a439812ee3","rayCluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:43:57.707Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"7532ae9c-8762-4573-88d9-b0a439812ee3","rayCluster":"counter-nl7mr","podIPs":[]}
{"level":"info","ts":"2026-06-05T09:43:57.708Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"5b37aa10-2047-453b-a614-932cd3004f74","cluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:43:57.805Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"d3da7747-4d5c-422f-9881-17726bd8bd23"}
{"level":"info","ts":"2026-06-05T09:43:57.806Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"d3da7747-4d5c-422f-9881-17726bd8bd23"}
{"level":"info","ts":"2026-06-05T09:43:57.806Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"b7d3a6e7-69b5-4958-96c9-2562b1cb1340","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:57.905Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"b7d3a6e7-69b5-4958-96c9-2562b1cb1340","name":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:43:57.806Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"d3da7747-4d5c-422f-9881-17726bd8bd23","1 head service found":"counter-nl7mr-head-svc"}
{"level":"info","ts":"2026-06-05T09:43:57.831Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"5b37aa10-2047-453b-a614-932cd3004f74","name":"counter-nl7mr-head"}
{"level":"info","ts":"2026-06-05T09:43:57.906Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"b7d3a6e7-69b5-4958-96c9-2562b1cb1340","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:43:57.906Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"d3da7747-4d5c-422f-9881-17726bd8bd23","Found 1 head Pod":"counter-nl7mr-head-7xdm5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:43:57.906Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"d3da7747-4d5c-422f-9881-17726bd8bd23","head Pod":"counter-nl7mr-head-7xdm5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-nl7mr-head-7xdm5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:57.906Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"d3da7747-4d5c-422f-9881-17726bd8bd23","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:43:57.906Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"d3da7747-4d5c-422f-9881-17726bd8bd23","worker Pod":"counter-nl7mr-small-group-worker-wpkdn","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-nl7mr-small-group-worker-wpkdn. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:57.906Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"d3da7747-4d5c-422f-9881-17726bd8bd23","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:57.906Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"d3da7747-4d5c-422f-9881-17726bd8bd23","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:43:57.906Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"d3da7747-4d5c-422f-9881-17726bd8bd23","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:57.906Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"b7d3a6e7-69b5-4958-96c9-2562b1cb1340"}
{"level":"info","ts":"2026-06-05T09:43:57.906Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"b7d3a6e7-69b5-4958-96c9-2562b1cb1340","RayCluster":"counter-nl7mr","State":""}
{"level":"info","ts":"2026-06-05T09:43:57.906Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"d3da7747-4d5c-422f-9881-17726bd8bd23","podName":"counter-nl7mr-head-7xdm5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:43:57.906Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"d3da7747-4d5c-422f-9881-17726bd8bd23","podName":"counter-nl7mr-head-7xdm5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:43:57.906Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"d3da7747-4d5c-422f-9881-17726bd8bd23","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"counter-nl7mr-head-svc"},"newHead":{"podName":"counter-nl7mr-head-7xdm5","serviceName":"counter-nl7mr-head-svc"}}
{"level":"info","ts":"2026-06-05T09:43:57.906Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"3245c004-e56b-4b9e-8aa4-f5784a727c34","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:57.906Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"3245c004-e56b-4b9e-8aa4-f5784a727c34","name":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:43:57.906Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"3245c004-e56b-4b9e-8aa4-f5784a727c34","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:43:57.906Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"d3da7747-4d5c-422f-9881-17726bd8bd23","name":"counter-nl7mr","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:43:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-nl7mr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:55Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:43:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-nl7mr-head-7xdm5","serviceName":"counter-nl7mr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:55Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:43:57.907Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"3245c004-e56b-4b9e-8aa4-f5784a727c34"}
{"level":"info","ts":"2026-06-05T09:43:57.907Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"3245c004-e56b-4b9e-8aa4-f5784a727c34","RayCluster":"counter-nl7mr","State":""}
{"level":"info","ts":"2026-06-05T09:43:57.917Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"d3da7747-4d5c-422f-9881-17726bd8bd23","name":"counter-nl7mr","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-nl7mr\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-thrrw","name":"counter-nl7mr"}}
{"level":"info","ts":"2026-06-05T09:43:57.917Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"d3da7747-4d5c-422f-9881-17726bd8bd23"}
{"level":"error","ts":"2026-06-05T09:43:57.917Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"d3da7747-4d5c-422f-9881-17726bd8bd23","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-nl7mr\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:43:57.917Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"016143a9-9b56-4aac-9fe1-b8ef344c1279"}
{"level":"info","ts":"2026-06-05T09:43:57.917Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"016143a9-9b56-4aac-9fe1-b8ef344c1279"}
{"level":"info","ts":"2026-06-05T09:43:57.917Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"016143a9-9b56-4aac-9fe1-b8ef344c1279","1 head service found":"counter-nl7mr-head-svc"}
{"level":"info","ts":"2026-06-05T09:43:57.917Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"016143a9-9b56-4aac-9fe1-b8ef344c1279","Found 1 head Pod":"counter-nl7mr-head-7xdm5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:43:57.917Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"016143a9-9b56-4aac-9fe1-b8ef344c1279","head Pod":"counter-nl7mr-head-7xdm5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-nl7mr-head-7xdm5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:57.917Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"016143a9-9b56-4aac-9fe1-b8ef344c1279","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:43:57.917Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"016143a9-9b56-4aac-9fe1-b8ef344c1279","worker Pod":"counter-nl7mr-small-group-worker-wpkdn","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-nl7mr-small-group-worker-wpkdn. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:57.917Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"016143a9-9b56-4aac-9fe1-b8ef344c1279","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:57.917Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"016143a9-9b56-4aac-9fe1-b8ef344c1279","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:43:57.917Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"016143a9-9b56-4aac-9fe1-b8ef344c1279","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:57.917Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"016143a9-9b56-4aac-9fe1-b8ef344c1279","podName":"counter-nl7mr-head-7xdm5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:43:57.917Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"016143a9-9b56-4aac-9fe1-b8ef344c1279","podName":"counter-nl7mr-head-7xdm5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:43:57.918Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"016143a9-9b56-4aac-9fe1-b8ef344c1279","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:55Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:55Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-05T09:43:57.918Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"016143a9-9b56-4aac-9fe1-b8ef344c1279","name":"counter-nl7mr","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:43:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-nl7mr-head-7xdm5","serviceName":"counter-nl7mr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:55Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:43:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-nl7mr-head-7xdm5","serviceName":"counter-nl7mr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:55Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:43:57.921Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"5b37aa10-2047-453b-a614-932cd3004f74","name":"counter-nl7mr-workers"}
{"level":"info","ts":"2026-06-05T09:43:57.921Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"5b37aa10-2047-453b-a614-932cd3004f74","cluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:43:58.007Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"06f9e900-3a91-48fb-84fd-f2d41e7a0635","rayCluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:43:58.007Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"4df0fc3c-2aaf-4434-a313-9e9c97a1f7c5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:58.007Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"4df0fc3c-2aaf-4434-a313-9e9c97a1f7c5","name":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:43:58.007Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"179f75b9-9b33-4a8f-a892-f07ebbec1ee2"}
{"level":"info","ts":"2026-06-05T09:43:58.007Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"86331059-ecea-4684-9ec6-a85145a746d7","cluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:43:58.007Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"179f75b9-9b33-4a8f-a892-f07ebbec1ee2"}
{"level":"info","ts":"2026-06-05T09:43:58.007Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"179f75b9-9b33-4a8f-a892-f07ebbec1ee2","1 head service found":"counter-nl7mr-head-svc"}
{"level":"info","ts":"2026-06-05T09:43:58.007Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"06f9e900-3a91-48fb-84fd-f2d41e7a0635","rayCluster":"counter-nl7mr","podIPs":[]}
{"level":"info","ts":"2026-06-05T09:43:58.007Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"179f75b9-9b33-4a8f-a892-f07ebbec1ee2","Found 1 head Pod":"counter-nl7mr-head-7xdm5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:43:58.007Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"179f75b9-9b33-4a8f-a892-f07ebbec1ee2","head Pod":"counter-nl7mr-head-7xdm5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-nl7mr-head-7xdm5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:58.007Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"179f75b9-9b33-4a8f-a892-f07ebbec1ee2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:43:58.008Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"179f75b9-9b33-4a8f-a892-f07ebbec1ee2","worker Pod":"counter-nl7mr-small-group-worker-wpkdn","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-nl7mr-small-group-worker-wpkdn. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:58.008Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"179f75b9-9b33-4a8f-a892-f07ebbec1ee2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:58.008Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"179f75b9-9b33-4a8f-a892-f07ebbec1ee2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:43:58.008Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"179f75b9-9b33-4a8f-a892-f07ebbec1ee2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:58.008Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"179f75b9-9b33-4a8f-a892-f07ebbec1ee2","podName":"counter-nl7mr-head-7xdm5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:43:58.008Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"179f75b9-9b33-4a8f-a892-f07ebbec1ee2","podName":"counter-nl7mr-head-7xdm5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:43:58.008Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"179f75b9-9b33-4a8f-a892-f07ebbec1ee2","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:43:58.008Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"179f75b9-9b33-4a8f-a892-f07ebbec1ee2","seconds":300}
{"level":"info","ts":"2026-06-05T09:43:58.013Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"4df0fc3c-2aaf-4434-a313-9e9c97a1f7c5","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:43:58.013Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"4df0fc3c-2aaf-4434-a313-9e9c97a1f7c5"}
{"level":"info","ts":"2026-06-05T09:43:58.013Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"4df0fc3c-2aaf-4434-a313-9e9c97a1f7c5","RayCluster":"counter-nl7mr","State":""}
{"level":"info","ts":"2026-06-05T09:43:58.022Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"86331059-ecea-4684-9ec6-a85145a746d7","name":"counter-nl7mr-head"}
{"level":"info","ts":"2026-06-05T09:43:58.033Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"86331059-ecea-4684-9ec6-a85145a746d7","name":"counter-nl7mr-workers"}
{"level":"info","ts":"2026-06-05T09:43:58.033Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"86331059-ecea-4684-9ec6-a85145a746d7","cluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:43:58.082Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"80df4770-8940-451b-9929-034796da0f7e"}
{"level":"info","ts":"2026-06-05T09:43:58.082Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"80df4770-8940-451b-9929-034796da0f7e"}
{"level":"info","ts":"2026-06-05T09:43:58.082Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"80df4770-8940-451b-9929-034796da0f7e","1 head service found":"counter-nl7mr-head-svc"}
{"level":"info","ts":"2026-06-05T09:43:58.082Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"80df4770-8940-451b-9929-034796da0f7e","Found 1 head Pod":"counter-nl7mr-head-7xdm5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:43:58.082Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"80df4770-8940-451b-9929-034796da0f7e","head Pod":"counter-nl7mr-head-7xdm5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-nl7mr-head-7xdm5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:58.082Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"80df4770-8940-451b-9929-034796da0f7e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:43:58.082Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"80df4770-8940-451b-9929-034796da0f7e","worker Pod":"counter-nl7mr-small-group-worker-wpkdn","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-nl7mr-small-group-worker-wpkdn. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:58.082Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"80df4770-8940-451b-9929-034796da0f7e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:58.082Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"80df4770-8940-451b-9929-034796da0f7e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:43:58.082Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"80df4770-8940-451b-9929-034796da0f7e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:58.082Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"80df4770-8940-451b-9929-034796da0f7e","podName":"counter-nl7mr-head-7xdm5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:43:58.082Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"80df4770-8940-451b-9929-034796da0f7e","podName":"counter-nl7mr-head-7xdm5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:43:58.083Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"80df4770-8940-451b-9929-034796da0f7e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:43:58.083Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"80df4770-8940-451b-9929-034796da0f7e","seconds":300}
{"level":"info","ts":"2026-06-05T09:43:58.105Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"b3dc988b-acad-41aa-a8ce-c830911a85e8"}
{"level":"info","ts":"2026-06-05T09:43:58.105Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"b3dc988b-acad-41aa-a8ce-c830911a85e8"}
{"level":"info","ts":"2026-06-05T09:43:58.105Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"b3dc988b-acad-41aa-a8ce-c830911a85e8","1 head service found":"counter-nl7mr-head-svc"}
{"level":"info","ts":"2026-06-05T09:43:58.105Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"b3dc988b-acad-41aa-a8ce-c830911a85e8","Found 1 head Pod":"counter-nl7mr-head-7xdm5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:43:58.105Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"b3dc988b-acad-41aa-a8ce-c830911a85e8","head Pod":"counter-nl7mr-head-7xdm5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-nl7mr-head-7xdm5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:58.105Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"b3dc988b-acad-41aa-a8ce-c830911a85e8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:43:58.105Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"b3dc988b-acad-41aa-a8ce-c830911a85e8","worker Pod":"counter-nl7mr-small-group-worker-wpkdn","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-nl7mr-small-group-worker-wpkdn. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:58.105Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"b3dc988b-acad-41aa-a8ce-c830911a85e8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:58.105Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"b3dc988b-acad-41aa-a8ce-c830911a85e8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:43:58.105Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"b3dc988b-acad-41aa-a8ce-c830911a85e8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:58.105Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"b3dc988b-acad-41aa-a8ce-c830911a85e8","podName":"counter-nl7mr-head-7xdm5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:43:58.105Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"b3dc988b-acad-41aa-a8ce-c830911a85e8","podName":"counter-nl7mr-head-7xdm5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:43:58.105Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"b3dc988b-acad-41aa-a8ce-c830911a85e8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:43:58.105Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"b3dc988b-acad-41aa-a8ce-c830911a85e8","seconds":300}
{"level":"info","ts":"2026-06-05T09:43:58.553Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"7ddfbe30-cd6a-4ef8-811d-2737da00f399"}
{"level":"info","ts":"2026-06-05T09:43:58.553Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"7ddfbe30-cd6a-4ef8-811d-2737da00f399"}
{"level":"info","ts":"2026-06-05T09:43:58.553Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"7ddfbe30-cd6a-4ef8-811d-2737da00f399","1 head service found":"counter-nl7mr-head-svc"}
{"level":"info","ts":"2026-06-05T09:43:58.554Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"7ddfbe30-cd6a-4ef8-811d-2737da00f399","Found 1 head Pod":"counter-nl7mr-head-7xdm5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:43:58.554Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"7ddfbe30-cd6a-4ef8-811d-2737da00f399","head Pod":"counter-nl7mr-head-7xdm5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-nl7mr-head-7xdm5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:58.554Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"7ddfbe30-cd6a-4ef8-811d-2737da00f399","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:43:58.554Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"7ddfbe30-cd6a-4ef8-811d-2737da00f399","worker Pod":"counter-nl7mr-small-group-worker-wpkdn","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-nl7mr-small-group-worker-wpkdn. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:58.554Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"7ddfbe30-cd6a-4ef8-811d-2737da00f399","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:58.554Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"7ddfbe30-cd6a-4ef8-811d-2737da00f399","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:43:58.554Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"7ddfbe30-cd6a-4ef8-811d-2737da00f399","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:58.554Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"7ddfbe30-cd6a-4ef8-811d-2737da00f399","podName":"counter-nl7mr-head-7xdm5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:43:58.554Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"7ddfbe30-cd6a-4ef8-811d-2737da00f399","podName":"counter-nl7mr-head-7xdm5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:43:58.554Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"7ddfbe30-cd6a-4ef8-811d-2737da00f399","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:43:58.554Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"7ddfbe30-cd6a-4ef8-811d-2737da00f399","seconds":300}
{"level":"info","ts":"2026-06-05T09:43:58.767Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"6816f77d-dcd1-4eb4-9e53-34d94abc8713"}
{"level":"info","ts":"2026-06-05T09:43:58.767Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"6816f77d-dcd1-4eb4-9e53-34d94abc8713"}
{"level":"info","ts":"2026-06-05T09:43:58.767Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"6816f77d-dcd1-4eb4-9e53-34d94abc8713","1 head service found":"counter-nl7mr-head-svc"}
{"level":"info","ts":"2026-06-05T09:43:58.767Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"6816f77d-dcd1-4eb4-9e53-34d94abc8713","Found 1 head Pod":"counter-nl7mr-head-7xdm5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:43:58.767Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"6816f77d-dcd1-4eb4-9e53-34d94abc8713","head Pod":"counter-nl7mr-head-7xdm5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-nl7mr-head-7xdm5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:58.767Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"6816f77d-dcd1-4eb4-9e53-34d94abc8713","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:43:58.767Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"6816f77d-dcd1-4eb4-9e53-34d94abc8713","worker Pod":"counter-nl7mr-small-group-worker-wpkdn","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-nl7mr-small-group-worker-wpkdn. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:43:58.767Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"6816f77d-dcd1-4eb4-9e53-34d94abc8713","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:58.767Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"6816f77d-dcd1-4eb4-9e53-34d94abc8713","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:43:58.767Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"6816f77d-dcd1-4eb4-9e53-34d94abc8713","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:43:58.767Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"6816f77d-dcd1-4eb4-9e53-34d94abc8713","podName":"counter-nl7mr-head-7xdm5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:43:58.767Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"6816f77d-dcd1-4eb4-9e53-34d94abc8713","podName":"counter-nl7mr-head-7xdm5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:43:58.767Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"6816f77d-dcd1-4eb4-9e53-34d94abc8713","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"counter-nl7mr-head-7xdm5","serviceName":"counter-nl7mr-head-svc"},"newHead":{"podIP":"10.132.0.22","serviceIP":"10.132.0.22","podName":"counter-nl7mr-head-7xdm5","serviceName":"counter-nl7mr-head-svc"}}
{"level":"info","ts":"2026-06-05T09:43:58.767Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"6816f77d-dcd1-4eb4-9e53-34d94abc8713","name":"counter-nl7mr","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:43:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-nl7mr-head-7xdm5","serviceName":"counter-nl7mr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:55Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:43:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.22","serviceIP":"10.132.0.22","podName":"counter-nl7mr-head-7xdm5","serviceName":"counter-nl7mr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:55Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:43:58.781Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"5056ada4-c16e-4b75-bc30-1a3925571cc9","rayCluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:43:58.781Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"2d196852-ff3f-4d04-9a60-0afb7d90de05","cluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:43:58.781Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"5056ada4-c16e-4b75-bc30-1a3925571cc9","pod":"counter-nl7mr-head-7xdm5","ip":"10.132.0.22"}
{"level":"info","ts":"2026-06-05T09:43:58.781Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"5056ada4-c16e-4b75-bc30-1a3925571cc9","pod":"counter-nl7mr-small-group-worker-wpkdn","ip":"10.133.0.43"}
{"level":"info","ts":"2026-06-05T09:43:58.781Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"c724b150-fd3f-41c6-bd8a-c1028ca6d4bb","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:43:58.781Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"c724b150-fd3f-41c6-bd8a-c1028ca6d4bb","name":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:43:58.781Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"c724b150-fd3f-41c6-bd8a-c1028ca6d4bb","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:43:58.781Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"c724b150-fd3f-41c6-bd8a-c1028ca6d4bb"}
{"level":"info","ts":"2026-06-05T09:43:58.781Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"c724b150-fd3f-41c6-bd8a-c1028ca6d4bb","RayCluster":"counter-nl7mr","State":""}
{"level":"info","ts":"2026-06-05T09:43:58.794Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"5056ada4-c16e-4b75-bc30-1a3925571cc9","certificate":"ray-head-cert-counter-nl7mr","dnsNames":["counter-nl7mr-head-svc","localhost","counter-nl7mr-head-svc.test-ns-thrrw.svc","counter-nl7mr-head-svc.test-ns-thrrw.svc.cluster.local"],"ipAddresses":["10.132.0.22","10.133.0.43","127.0.0.1"]}
{"level":"info","ts":"2026-06-05T09:43:58.795Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"2d196852-ff3f-4d04-9a60-0afb7d90de05","name":"counter-nl7mr-head"}
{"level":"info","ts":"2026-06-05T09:43:58.809Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"5056ada4-c16e-4b75-bc30-1a3925571cc9","certificate":"ray-worker-cert-counter-nl7mr","dnsNames":["counter-nl7mr-worker-svc","localhost","counter-nl7mr-small-group","counter-nl7mr-worker-svc.test-ns-thrrw.svc","counter-nl7mr-worker-svc.test-ns-thrrw.svc.cluster.local","*.counter-nl7mr-worker-svc.test-ns-thrrw.svc","*.counter-nl7mr-worker-svc.test-ns-thrrw.svc.cluster.local","*-worker-*.test-ns-thrrw.svc","*-worker-*.test-ns-thrrw.svc.cluster.local"],"ipAddresses":["10.132.0.22","10.133.0.43","127.0.0.1"]}
{"level":"info","ts":"2026-06-05T09:43:58.809Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"5056ada4-c16e-4b75-bc30-1a3925571cc9","rayCluster":"counter-nl7mr","podIPs":["10.132.0.22","10.133.0.43"]}
{"level":"info","ts":"2026-06-05T09:43:58.816Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"2d196852-ff3f-4d04-9a60-0afb7d90de05","name":"counter-nl7mr-workers"}
{"level":"info","ts":"2026-06-05T09:43:58.816Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"2d196852-ff3f-4d04-9a60-0afb7d90de05","cluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:00.008Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"d91f1d8d-a90f-4500-a05d-069047faaa46"}
{"level":"info","ts":"2026-06-05T09:44:00.008Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"d91f1d8d-a90f-4500-a05d-069047faaa46"}
{"level":"info","ts":"2026-06-05T09:44:00.008Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"d91f1d8d-a90f-4500-a05d-069047faaa46","1 head service found":"counter-nl7mr-head-svc"}
{"level":"info","ts":"2026-06-05T09:44:00.008Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"d91f1d8d-a90f-4500-a05d-069047faaa46","Found 1 head Pod":"counter-nl7mr-head-7xdm5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:44:00.008Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"d91f1d8d-a90f-4500-a05d-069047faaa46","head Pod":"counter-nl7mr-head-7xdm5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-nl7mr-head-7xdm5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:44:00.008Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"d91f1d8d-a90f-4500-a05d-069047faaa46","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:44:00.008Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"d91f1d8d-a90f-4500-a05d-069047faaa46","worker Pod":"counter-nl7mr-small-group-worker-wpkdn","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-nl7mr-small-group-worker-wpkdn. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:44:00.008Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"d91f1d8d-a90f-4500-a05d-069047faaa46","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:44:00.008Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"d91f1d8d-a90f-4500-a05d-069047faaa46","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:44:00.008Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"d91f1d8d-a90f-4500-a05d-069047faaa46","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:44:00.008Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"d91f1d8d-a90f-4500-a05d-069047faaa46","podName":"counter-nl7mr-head-7xdm5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:44:00.008Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"d91f1d8d-a90f-4500-a05d-069047faaa46","podName":"counter-nl7mr-head-7xdm5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:44:00.008Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"d91f1d8d-a90f-4500-a05d-069047faaa46","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:44:00.009Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"d91f1d8d-a90f-4500-a05d-069047faaa46","seconds":300}
{"level":"info","ts":"2026-06-05T09:44:00.907Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"a5aee17b-397b-400d-8acf-a95e6c9dd8b0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:44:00.907Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"a5aee17b-397b-400d-8acf-a95e6c9dd8b0","name":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:00.907Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"a5aee17b-397b-400d-8acf-a95e6c9dd8b0","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:44:00.908Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"a5aee17b-397b-400d-8acf-a95e6c9dd8b0"}
{"level":"info","ts":"2026-06-05T09:44:00.908Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"a5aee17b-397b-400d-8acf-a95e6c9dd8b0","RayCluster":"counter-nl7mr","State":""}
{"level":"info","ts":"2026-06-05T09:44:03.783Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"18e3cce3-f208-4af4-ae8c-e9ae6fd97d74"}
{"level":"info","ts":"2026-06-05T09:44:03.783Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"18e3cce3-f208-4af4-ae8c-e9ae6fd97d74"}
{"level":"info","ts":"2026-06-05T09:44:03.783Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"18e3cce3-f208-4af4-ae8c-e9ae6fd97d74","1 head service found":"counter-nl7mr-head-svc"}
{"level":"info","ts":"2026-06-05T09:44:03.783Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"18e3cce3-f208-4af4-ae8c-e9ae6fd97d74","Found 1 head Pod":"counter-nl7mr-head-7xdm5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:44:03.783Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"18e3cce3-f208-4af4-ae8c-e9ae6fd97d74","head Pod":"counter-nl7mr-head-7xdm5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-nl7mr-head-7xdm5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:44:03.783Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"18e3cce3-f208-4af4-ae8c-e9ae6fd97d74","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:44:03.783Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"18e3cce3-f208-4af4-ae8c-e9ae6fd97d74","worker Pod":"counter-nl7mr-small-group-worker-wpkdn","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-nl7mr-small-group-worker-wpkdn. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:44:03.783Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"18e3cce3-f208-4af4-ae8c-e9ae6fd97d74","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:44:03.783Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"18e3cce3-f208-4af4-ae8c-e9ae6fd97d74","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:44:03.783Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"18e3cce3-f208-4af4-ae8c-e9ae6fd97d74","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:44:03.783Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"18e3cce3-f208-4af4-ae8c-e9ae6fd97d74","podName":"counter-nl7mr-head-7xdm5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:44:03.783Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"18e3cce3-f208-4af4-ae8c-e9ae6fd97d74","podName":"counter-nl7mr-head-7xdm5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:44:03.783Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"18e3cce3-f208-4af4-ae8c-e9ae6fd97d74","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:44:03.783Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"18e3cce3-f208-4af4-ae8c-e9ae6fd97d74","seconds":300}
{"level":"info","ts":"2026-06-05T09:44:03.909Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"9b3b61b0-906d-4a1d-b5d1-e616e41ce3e8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:44:03.909Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"9b3b61b0-906d-4a1d-b5d1-e616e41ce3e8","name":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:03.909Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"9b3b61b0-906d-4a1d-b5d1-e616e41ce3e8","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:44:03.909Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"9b3b61b0-906d-4a1d-b5d1-e616e41ce3e8"}
{"level":"info","ts":"2026-06-05T09:44:03.909Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"9b3b61b0-906d-4a1d-b5d1-e616e41ce3e8","RayCluster":"counter-nl7mr","State":""}
{"level":"info","ts":"2026-06-05T09:44:04.000Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"3cd37379-1bda-4063-979b-aecc27e99ab3"}
{"level":"info","ts":"2026-06-05T09:44:04.000Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"3cd37379-1bda-4063-979b-aecc27e99ab3"}
{"level":"info","ts":"2026-06-05T09:44:04.000Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"3cd37379-1bda-4063-979b-aecc27e99ab3","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-05T09:44:04.000Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"3cd37379-1bda-4063-979b-aecc27e99ab3","Found 1 head Pod":"raycluster-head-vjd7d","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:44:04.000Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"3cd37379-1bda-4063-979b-aecc27e99ab3","head Pod":"raycluster-head-vjd7d","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-vjd7d. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:44:04.000Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"3cd37379-1bda-4063-979b-aecc27e99ab3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:44:04.000Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"3cd37379-1bda-4063-979b-aecc27e99ab3","worker Pod":"raycluster-small-group-worker-j9pbw","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-j9pbw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:44:04.000Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"3cd37379-1bda-4063-979b-aecc27e99ab3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:44:04.000Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"3cd37379-1bda-4063-979b-aecc27e99ab3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:44:04.000Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"3cd37379-1bda-4063-979b-aecc27e99ab3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:44:04.000Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"3cd37379-1bda-4063-979b-aecc27e99ab3","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:44:04.000Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"3cd37379-1bda-4063-979b-aecc27e99ab3","seconds":300}
{"level":"info","ts":"2026-06-05T09:44:04.789Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"b312b77a-58a6-4a67-98a1-a35a9eecdd33"}
{"level":"info","ts":"2026-06-05T09:44:04.789Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"b312b77a-58a6-4a67-98a1-a35a9eecdd33"}
{"level":"info","ts":"2026-06-05T09:44:04.789Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"b312b77a-58a6-4a67-98a1-a35a9eecdd33","1 head service found":"counter-nl7mr-head-svc"}
{"level":"info","ts":"2026-06-05T09:44:04.789Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"b312b77a-58a6-4a67-98a1-a35a9eecdd33","Found 1 head Pod":"counter-nl7mr-head-7xdm5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:44:04.789Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"b312b77a-58a6-4a67-98a1-a35a9eecdd33","head Pod":"counter-nl7mr-head-7xdm5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-nl7mr-head-7xdm5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:44:04.789Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"b312b77a-58a6-4a67-98a1-a35a9eecdd33","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:44:04.789Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"b312b77a-58a6-4a67-98a1-a35a9eecdd33","worker Pod":"counter-nl7mr-small-group-worker-wpkdn","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-nl7mr-small-group-worker-wpkdn. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:44:04.789Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"b312b77a-58a6-4a67-98a1-a35a9eecdd33","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:44:04.789Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"b312b77a-58a6-4a67-98a1-a35a9eecdd33","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:44:04.789Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"b312b77a-58a6-4a67-98a1-a35a9eecdd33","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:44:04.789Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"b312b77a-58a6-4a67-98a1-a35a9eecdd33","podName":"counter-nl7mr-head-7xdm5","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:43:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:44:04.789Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"b312b77a-58a6-4a67-98a1-a35a9eecdd33","podName":"counter-nl7mr-head-7xdm5","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:43:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:44:04.789Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"b312b77a-58a6-4a67-98a1-a35a9eecdd33","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:55Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:55Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-05T09:44:04.789Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"b312b77a-58a6-4a67-98a1-a35a9eecdd33","name":"counter-nl7mr","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:43:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.22","serviceIP":"10.132.0.22","podName":"counter-nl7mr-head-7xdm5","serviceName":"counter-nl7mr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:55Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:44:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.22","serviceIP":"10.132.0.22","podName":"counter-nl7mr-head-7xdm5","serviceName":"counter-nl7mr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:55Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:44:04.802Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"bc94c931-c757-4f2c-80a2-31b880540d59","cluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:04.802Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"3c95f40a-0cf1-4738-af35-f6f10abd1280","rayCluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:04.802Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"a68c6276-cffa-4123-ae2f-03f538bb4edb","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:44:04.802Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"a68c6276-cffa-4123-ae2f-03f538bb4edb","name":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:04.802Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"3c95f40a-0cf1-4738-af35-f6f10abd1280","pod":"counter-nl7mr-head-7xdm5","ip":"10.132.0.22"}
{"level":"info","ts":"2026-06-05T09:44:04.802Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"3c95f40a-0cf1-4738-af35-f6f10abd1280","pod":"counter-nl7mr-small-group-worker-wpkdn","ip":"10.133.0.43"}
{"level":"info","ts":"2026-06-05T09:44:04.802Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"3c95f40a-0cf1-4738-af35-f6f10abd1280","rayCluster":"counter-nl7mr","podIPs":["10.132.0.22","10.133.0.43"]}
{"level":"info","ts":"2026-06-05T09:44:04.802Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"a68c6276-cffa-4123-ae2f-03f538bb4edb","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:44:04.803Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"a68c6276-cffa-4123-ae2f-03f538bb4edb"}
{"level":"info","ts":"2026-06-05T09:44:04.803Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"a68c6276-cffa-4123-ae2f-03f538bb4edb","RayCluster":"counter-nl7mr","State":""}
{"level":"info","ts":"2026-06-05T09:44:04.816Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"bc94c931-c757-4f2c-80a2-31b880540d59","name":"counter-nl7mr-head"}
{"level":"info","ts":"2026-06-05T09:44:04.827Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"bc94c931-c757-4f2c-80a2-31b880540d59","name":"counter-nl7mr-workers"}
{"level":"info","ts":"2026-06-05T09:44:04.827Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"bc94c931-c757-4f2c-80a2-31b880540d59","cluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:06.802Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"35213d74-e439-4a3f-9845-3061c7a7198d"}
{"level":"info","ts":"2026-06-05T09:44:06.802Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"35213d74-e439-4a3f-9845-3061c7a7198d"}
{"level":"info","ts":"2026-06-05T09:44:06.803Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"35213d74-e439-4a3f-9845-3061c7a7198d","1 head service found":"counter-nl7mr-head-svc"}
{"level":"info","ts":"2026-06-05T09:44:06.803Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"35213d74-e439-4a3f-9845-3061c7a7198d","Found 1 head Pod":"counter-nl7mr-head-7xdm5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:44:06.803Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"35213d74-e439-4a3f-9845-3061c7a7198d","head Pod":"counter-nl7mr-head-7xdm5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-nl7mr-head-7xdm5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:44:06.803Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"35213d74-e439-4a3f-9845-3061c7a7198d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:44:06.803Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"35213d74-e439-4a3f-9845-3061c7a7198d","worker Pod":"counter-nl7mr-small-group-worker-wpkdn","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-nl7mr-small-group-worker-wpkdn. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:44:06.803Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"35213d74-e439-4a3f-9845-3061c7a7198d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:44:06.803Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"35213d74-e439-4a3f-9845-3061c7a7198d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:44:06.803Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"35213d74-e439-4a3f-9845-3061c7a7198d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:44:06.803Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"35213d74-e439-4a3f-9845-3061c7a7198d","podName":"counter-nl7mr-head-7xdm5","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:43:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:44:06.803Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"35213d74-e439-4a3f-9845-3061c7a7198d","podName":"counter-nl7mr-head-7xdm5","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:43:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:44:06.803Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"35213d74-e439-4a3f-9845-3061c7a7198d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:44:06.803Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"35213d74-e439-4a3f-9845-3061c7a7198d","seconds":300}
{"level":"info","ts":"2026-06-05T09:44:06.910Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"c54c502a-e8f0-46e5-9bc2-7a1e14875c54","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:44:06.910Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"c54c502a-e8f0-46e5-9bc2-7a1e14875c54","name":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:06.910Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"c54c502a-e8f0-46e5-9bc2-7a1e14875c54","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:44:06.910Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"c54c502a-e8f0-46e5-9bc2-7a1e14875c54"}
{"level":"info","ts":"2026-06-05T09:44:06.910Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"c54c502a-e8f0-46e5-9bc2-7a1e14875c54","RayCluster":"counter-nl7mr","State":""}
{"level":"info","ts":"2026-06-05T09:44:08.590Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"8e373539-9bbf-4826-96c8-5712f7222d0b"}
{"level":"info","ts":"2026-06-05T09:44:08.590Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"8e373539-9bbf-4826-96c8-5712f7222d0b"}
{"level":"info","ts":"2026-06-05T09:44:08.590Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"8e373539-9bbf-4826-96c8-5712f7222d0b","1 head service found":"counter-nl7mr-head-svc"}
{"level":"info","ts":"2026-06-05T09:44:08.590Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"8e373539-9bbf-4826-96c8-5712f7222d0b","Found 1 head Pod":"counter-nl7mr-head-7xdm5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:44:08.590Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"8e373539-9bbf-4826-96c8-5712f7222d0b","head Pod":"counter-nl7mr-head-7xdm5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-nl7mr-head-7xdm5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:44:08.590Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"8e373539-9bbf-4826-96c8-5712f7222d0b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:44:08.590Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"8e373539-9bbf-4826-96c8-5712f7222d0b","worker Pod":"counter-nl7mr-small-group-worker-wpkdn","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-nl7mr-small-group-worker-wpkdn. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:44:08.590Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"8e373539-9bbf-4826-96c8-5712f7222d0b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:44:08.590Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"8e373539-9bbf-4826-96c8-5712f7222d0b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:44:08.590Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"8e373539-9bbf-4826-96c8-5712f7222d0b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:44:08.591Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"8e373539-9bbf-4826-96c8-5712f7222d0b","podName":"counter-nl7mr-head-7xdm5","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:43:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:44:08.591Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"8e373539-9bbf-4826-96c8-5712f7222d0b","podName":"counter-nl7mr-head-7xdm5","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:43:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:44:08.591Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"8e373539-9bbf-4826-96c8-5712f7222d0b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:44:08.591Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"8e373539-9bbf-4826-96c8-5712f7222d0b","seconds":300}
{"level":"info","ts":"2026-06-05T09:44:09.595Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"34f5b6ba-9430-4781-9a09-c46b236f99b7"}
{"level":"info","ts":"2026-06-05T09:44:09.595Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"34f5b6ba-9430-4781-9a09-c46b236f99b7"}
{"level":"info","ts":"2026-06-05T09:44:09.595Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"34f5b6ba-9430-4781-9a09-c46b236f99b7","1 head service found":"counter-nl7mr-head-svc"}
{"level":"info","ts":"2026-06-05T09:44:09.595Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"34f5b6ba-9430-4781-9a09-c46b236f99b7","Found 1 head Pod":"counter-nl7mr-head-7xdm5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:44:09.595Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"34f5b6ba-9430-4781-9a09-c46b236f99b7","head Pod":"counter-nl7mr-head-7xdm5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-nl7mr-head-7xdm5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:44:09.595Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"34f5b6ba-9430-4781-9a09-c46b236f99b7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:44:09.595Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"34f5b6ba-9430-4781-9a09-c46b236f99b7","worker Pod":"counter-nl7mr-small-group-worker-wpkdn","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-nl7mr-small-group-worker-wpkdn. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:44:09.595Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"34f5b6ba-9430-4781-9a09-c46b236f99b7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:44:09.595Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"34f5b6ba-9430-4781-9a09-c46b236f99b7","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:44:09.595Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"34f5b6ba-9430-4781-9a09-c46b236f99b7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:44:09.595Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"34f5b6ba-9430-4781-9a09-c46b236f99b7","podName":"counter-nl7mr-head-7xdm5","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:43:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:44:09.595Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"34f5b6ba-9430-4781-9a09-c46b236f99b7","podName":"counter-nl7mr-head-7xdm5","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:43:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:44:09.595Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"34f5b6ba-9430-4781-9a09-c46b236f99b7","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-05T09:44:09.595Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"34f5b6ba-9430-4781-9a09-c46b236f99b7","name":"counter-nl7mr","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:44:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.22","serviceIP":"10.132.0.22","podName":"counter-nl7mr-head-7xdm5","serviceName":"counter-nl7mr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:55Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:44:09Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.22","serviceIP":"10.132.0.22","podName":"counter-nl7mr-head-7xdm5","serviceName":"counter-nl7mr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:55Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:44:09.608Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"505748ba-7b71-465b-91db-74e30a4474a3","rayCluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:09.608Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"6207e200-6c64-4ebe-b1c0-c516144d2716","cluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:09.608Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"505748ba-7b71-465b-91db-74e30a4474a3","pod":"counter-nl7mr-head-7xdm5","ip":"10.132.0.22"}
{"level":"info","ts":"2026-06-05T09:44:09.608Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"505748ba-7b71-465b-91db-74e30a4474a3","pod":"counter-nl7mr-small-group-worker-wpkdn","ip":"10.133.0.43"}
{"level":"info","ts":"2026-06-05T09:44:09.608Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"3f837130-91e1-4bc2-9857-3d075a0c20d8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:44:09.608Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"3f837130-91e1-4bc2-9857-3d075a0c20d8","name":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:09.608Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"3f837130-91e1-4bc2-9857-3d075a0c20d8","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:44:09.608Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"505748ba-7b71-465b-91db-74e30a4474a3","rayCluster":"counter-nl7mr","podIPs":["10.132.0.22","10.133.0.43"]}
{"level":"info","ts":"2026-06-05T09:44:09.609Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"3f837130-91e1-4bc2-9857-3d075a0c20d8"}
{"level":"info","ts":"2026-06-05T09:44:09.609Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"3f837130-91e1-4bc2-9857-3d075a0c20d8","RayCluster":"counter-nl7mr","State":""}
{"level":"info","ts":"2026-06-05T09:44:09.620Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"6207e200-6c64-4ebe-b1c0-c516144d2716","name":"counter-nl7mr-head"}
{"level":"info","ts":"2026-06-05T09:44:09.633Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"6207e200-6c64-4ebe-b1c0-c516144d2716","name":"counter-nl7mr-workers"}
{"level":"info","ts":"2026-06-05T09:44:09.633Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"6207e200-6c64-4ebe-b1c0-c516144d2716","cluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:09.911Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"b74a277f-cb6c-48ca-913c-6ae645a920f1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:44:09.911Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"b74a277f-cb6c-48ca-913c-6ae645a920f1","name":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:09.911Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"b74a277f-cb6c-48ca-913c-6ae645a920f1","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:44:09.911Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"b74a277f-cb6c-48ca-913c-6ae645a920f1"}
{"level":"info","ts":"2026-06-05T09:44:09.911Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"b74a277f-cb6c-48ca-913c-6ae645a920f1","RayCluster":"counter-nl7mr","State":""}
{"level":"info","ts":"2026-06-05T09:44:11.608Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"b978b48b-e124-49eb-806b-53d9dc40be5e"}
{"level":"info","ts":"2026-06-05T09:44:11.609Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"b978b48b-e124-49eb-806b-53d9dc40be5e"}
{"level":"info","ts":"2026-06-05T09:44:11.609Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"b978b48b-e124-49eb-806b-53d9dc40be5e","1 head service found":"counter-nl7mr-head-svc"}
{"level":"info","ts":"2026-06-05T09:44:11.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"b978b48b-e124-49eb-806b-53d9dc40be5e","Found 1 head Pod":"counter-nl7mr-head-7xdm5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:44:11.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"b978b48b-e124-49eb-806b-53d9dc40be5e","head Pod":"counter-nl7mr-head-7xdm5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-nl7mr-head-7xdm5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:44:11.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"b978b48b-e124-49eb-806b-53d9dc40be5e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:44:11.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"b978b48b-e124-49eb-806b-53d9dc40be5e","worker Pod":"counter-nl7mr-small-group-worker-wpkdn","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-nl7mr-small-group-worker-wpkdn. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:44:11.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"b978b48b-e124-49eb-806b-53d9dc40be5e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:44:11.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"b978b48b-e124-49eb-806b-53d9dc40be5e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:44:11.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"b978b48b-e124-49eb-806b-53d9dc40be5e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:44:11.609Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"b978b48b-e124-49eb-806b-53d9dc40be5e","podName":"counter-nl7mr-head-7xdm5","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:43:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:44:11.609Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"b978b48b-e124-49eb-806b-53d9dc40be5e","podName":"counter-nl7mr-head-7xdm5","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:43:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:44:11.609Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"b978b48b-e124-49eb-806b-53d9dc40be5e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:44:11.609Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"b978b48b-e124-49eb-806b-53d9dc40be5e","seconds":300}
{"level":"info","ts":"2026-06-05T09:44:12.912Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"0ff2b584-6a14-4b43-ba65-d8c47585bf00","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:44:12.912Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"0ff2b584-6a14-4b43-ba65-d8c47585bf00","name":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:12.912Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"0ff2b584-6a14-4b43-ba65-d8c47585bf00","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:44:12.912Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"0ff2b584-6a14-4b43-ba65-d8c47585bf00"}
{"level":"info","ts":"2026-06-05T09:44:12.912Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"0ff2b584-6a14-4b43-ba65-d8c47585bf00","RayCluster":"counter-nl7mr","State":""}
{"level":"info","ts":"2026-06-05T09:44:15.818Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"ed2a2bd4-78c6-4758-90f1-89f5dc6497f4"}
{"level":"info","ts":"2026-06-05T09:44:15.818Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"ed2a2bd4-78c6-4758-90f1-89f5dc6497f4"}
{"level":"info","ts":"2026-06-05T09:44:15.818Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"ed2a2bd4-78c6-4758-90f1-89f5dc6497f4","1 head service found":"counter-nl7mr-head-svc"}
{"level":"info","ts":"2026-06-05T09:44:15.818Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"ed2a2bd4-78c6-4758-90f1-89f5dc6497f4","Found 1 head Pod":"counter-nl7mr-head-7xdm5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:44:15.818Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"ed2a2bd4-78c6-4758-90f1-89f5dc6497f4","head Pod":"counter-nl7mr-head-7xdm5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-nl7mr-head-7xdm5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:44:15.818Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"ed2a2bd4-78c6-4758-90f1-89f5dc6497f4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:44:15.819Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"ed2a2bd4-78c6-4758-90f1-89f5dc6497f4","worker Pod":"counter-nl7mr-small-group-worker-wpkdn","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-nl7mr-small-group-worker-wpkdn. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:44:15.819Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"ed2a2bd4-78c6-4758-90f1-89f5dc6497f4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:44:15.819Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"ed2a2bd4-78c6-4758-90f1-89f5dc6497f4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:44:15.819Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"ed2a2bd4-78c6-4758-90f1-89f5dc6497f4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:44:15.819Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"ed2a2bd4-78c6-4758-90f1-89f5dc6497f4","podName":"counter-nl7mr-small-group-worker-wpkdn","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:43:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-05T09:44:15.819Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"ed2a2bd4-78c6-4758-90f1-89f5dc6497f4","podName":"counter-nl7mr-small-group-worker-wpkdn","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:43:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-05T09:44:15.819Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"ed2a2bd4-78c6-4758-90f1-89f5dc6497f4","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:55Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:44:15Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:55Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-05T09:44:15.819Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"ed2a2bd4-78c6-4758-90f1-89f5dc6497f4","name":"counter-nl7mr","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:44:09Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.22","serviceIP":"10.132.0.22","podName":"counter-nl7mr-head-7xdm5","serviceName":"counter-nl7mr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:55Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:44:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.22","serviceIP":"10.132.0.22","podName":"counter-nl7mr-head-7xdm5","serviceName":"counter-nl7mr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:44:15Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:55Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:44:15.832Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"694d46d6-b3fa-4c98-bd6a-55797c14e776","cluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:15.832Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"fa59e8cf-c7b9-49f9-922a-3df0722b3f7c","rayCluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:15.832Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"f06242b7-adb2-45d1-b0db-118830c10cb1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:44:15.832Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"f06242b7-adb2-45d1-b0db-118830c10cb1","name":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:15.832Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"f06242b7-adb2-45d1-b0db-118830c10cb1","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:44:15.832Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"fa59e8cf-c7b9-49f9-922a-3df0722b3f7c","pod":"counter-nl7mr-head-7xdm5","ip":"10.132.0.22"}
{"level":"info","ts":"2026-06-05T09:44:15.832Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"fa59e8cf-c7b9-49f9-922a-3df0722b3f7c","pod":"counter-nl7mr-small-group-worker-wpkdn","ip":"10.133.0.43"}
{"level":"info","ts":"2026-06-05T09:44:15.832Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"fa59e8cf-c7b9-49f9-922a-3df0722b3f7c","rayCluster":"counter-nl7mr","podIPs":["10.132.0.22","10.133.0.43"]}
{"level":"info","ts":"2026-06-05T09:44:15.832Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"f06242b7-adb2-45d1-b0db-118830c10cb1"}
{"level":"info","ts":"2026-06-05T09:44:15.832Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"f06242b7-adb2-45d1-b0db-118830c10cb1","RayCluster":"counter-nl7mr","State":""}
{"level":"info","ts":"2026-06-05T09:44:15.853Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"694d46d6-b3fa-4c98-bd6a-55797c14e776","name":"counter-nl7mr-head"}
{"level":"info","ts":"2026-06-05T09:44:15.863Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"694d46d6-b3fa-4c98-bd6a-55797c14e776","name":"counter-nl7mr-workers"}
{"level":"info","ts":"2026-06-05T09:44:15.863Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"694d46d6-b3fa-4c98-bd6a-55797c14e776","cluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:15.913Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"11d9fd5e-8114-4a03-b079-4749327bc6fc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:44:15.913Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"11d9fd5e-8114-4a03-b079-4749327bc6fc","name":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:15.913Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"11d9fd5e-8114-4a03-b079-4749327bc6fc","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:44:15.913Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"11d9fd5e-8114-4a03-b079-4749327bc6fc"}
{"level":"info","ts":"2026-06-05T09:44:15.913Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"11d9fd5e-8114-4a03-b079-4749327bc6fc","RayCluster":"counter-nl7mr","State":""}
{"level":"info","ts":"2026-06-05T09:44:17.832Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"8b3e4ed0-2054-4a9e-a4d1-ca69351551ad"}
{"level":"info","ts":"2026-06-05T09:44:17.832Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"8b3e4ed0-2054-4a9e-a4d1-ca69351551ad"}
{"level":"info","ts":"2026-06-05T09:44:17.832Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"8b3e4ed0-2054-4a9e-a4d1-ca69351551ad","1 head service found":"counter-nl7mr-head-svc"}
{"level":"info","ts":"2026-06-05T09:44:17.832Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"8b3e4ed0-2054-4a9e-a4d1-ca69351551ad","Found 1 head Pod":"counter-nl7mr-head-7xdm5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:44:17.832Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"8b3e4ed0-2054-4a9e-a4d1-ca69351551ad","head Pod":"counter-nl7mr-head-7xdm5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-nl7mr-head-7xdm5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:44:17.832Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"8b3e4ed0-2054-4a9e-a4d1-ca69351551ad","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:44:17.832Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"8b3e4ed0-2054-4a9e-a4d1-ca69351551ad","worker Pod":"counter-nl7mr-small-group-worker-wpkdn","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-nl7mr-small-group-worker-wpkdn. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:44:17.832Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"8b3e4ed0-2054-4a9e-a4d1-ca69351551ad","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:44:17.832Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"8b3e4ed0-2054-4a9e-a4d1-ca69351551ad","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:44:17.832Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"8b3e4ed0-2054-4a9e-a4d1-ca69351551ad","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:44:17.833Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"8b3e4ed0-2054-4a9e-a4d1-ca69351551ad","podName":"counter-nl7mr-small-group-worker-wpkdn","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:43:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-05T09:44:17.833Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"8b3e4ed0-2054-4a9e-a4d1-ca69351551ad","podName":"counter-nl7mr-small-group-worker-wpkdn","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:43:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-05T09:44:17.833Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"8b3e4ed0-2054-4a9e-a4d1-ca69351551ad","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:44:17.833Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"8b3e4ed0-2054-4a9e-a4d1-ca69351551ad","seconds":300}
{"level":"info","ts":"2026-06-05T09:44:18.914Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"f83c4f57-446b-445d-a0b9-e9a3150ca2c0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:44:18.914Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"f83c4f57-446b-445d-a0b9-e9a3150ca2c0","name":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:18.914Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"f83c4f57-446b-445d-a0b9-e9a3150ca2c0","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:44:18.914Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"f83c4f57-446b-445d-a0b9-e9a3150ca2c0"}
{"level":"info","ts":"2026-06-05T09:44:18.914Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"f83c4f57-446b-445d-a0b9-e9a3150ca2c0","RayCluster":"counter-nl7mr","State":""}
{"level":"info","ts":"2026-06-05T09:44:20.620Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"04ae8c9e-3cf0-4ae1-9f8c-c9e63a34940d"}
{"level":"info","ts":"2026-06-05T09:44:20.620Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"04ae8c9e-3cf0-4ae1-9f8c-c9e63a34940d"}
{"level":"info","ts":"2026-06-05T09:44:20.620Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"04ae8c9e-3cf0-4ae1-9f8c-c9e63a34940d","1 head service found":"counter-nl7mr-head-svc"}
{"level":"info","ts":"2026-06-05T09:44:20.620Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"04ae8c9e-3cf0-4ae1-9f8c-c9e63a34940d","Found 1 head Pod":"counter-nl7mr-head-7xdm5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:44:20.620Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"04ae8c9e-3cf0-4ae1-9f8c-c9e63a34940d","head Pod":"counter-nl7mr-head-7xdm5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-nl7mr-head-7xdm5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:44:20.620Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"04ae8c9e-3cf0-4ae1-9f8c-c9e63a34940d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:44:20.620Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"04ae8c9e-3cf0-4ae1-9f8c-c9e63a34940d","worker Pod":"counter-nl7mr-small-group-worker-wpkdn","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-nl7mr-small-group-worker-wpkdn. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:44:20.620Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"04ae8c9e-3cf0-4ae1-9f8c-c9e63a34940d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:44:20.620Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"04ae8c9e-3cf0-4ae1-9f8c-c9e63a34940d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:44:20.620Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"04ae8c9e-3cf0-4ae1-9f8c-c9e63a34940d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:44:20.621Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"04ae8c9e-3cf0-4ae1-9f8c-c9e63a34940d","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-06-05T09:44:20.621Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"04ae8c9e-3cf0-4ae1-9f8c-c9e63a34940d","name":"counter-nl7mr","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:44:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.22","serviceIP":"10.132.0.22","podName":"counter-nl7mr-head-7xdm5","serviceName":"counter-nl7mr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:44:15Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:55Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:44:20Z","stateTransitionTimes":{"ready":"2026-06-05T09:44:20Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.22","serviceIP":"10.132.0.22","podName":"counter-nl7mr-head-7xdm5","serviceName":"counter-nl7mr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:44:15Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:44:20Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:55Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:44:20.633Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"2e75ebae-2a6b-4414-8b23-0c67e4990188","cluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:20.633Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"49ede80e-7fca-4aae-b998-cc98e6c4c411","rayCluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:20.634Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"397d99c3-8cd3-4611-bdcb-797913d62af3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:44:20.634Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"397d99c3-8cd3-4611-bdcb-797913d62af3","name":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:20.634Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"49ede80e-7fca-4aae-b998-cc98e6c4c411","pod":"counter-nl7mr-head-7xdm5","ip":"10.132.0.22"}
{"level":"info","ts":"2026-06-05T09:44:20.634Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"49ede80e-7fca-4aae-b998-cc98e6c4c411","pod":"counter-nl7mr-small-group-worker-wpkdn","ip":"10.133.0.43"}
{"level":"info","ts":"2026-06-05T09:44:20.634Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"397d99c3-8cd3-4611-bdcb-797913d62af3","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:44:20.634Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"49ede80e-7fca-4aae-b998-cc98e6c4c411","rayCluster":"counter-nl7mr","podIPs":["10.132.0.22","10.133.0.43"]}
{"level":"info","ts":"2026-06-05T09:44:20.634Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"397d99c3-8cd3-4611-bdcb-797913d62af3"}
{"level":"info","ts":"2026-06-05T09:44:20.634Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"397d99c3-8cd3-4611-bdcb-797913d62af3","head service name":"counter-nl7mr-head-svc","namespace":"test-ns-thrrw"}
{"level":"info","ts":"2026-06-05T09:44:20.634Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"397d99c3-8cd3-4611-bdcb-797913d62af3","head service URL":"counter-nl7mr-head-svc.test-ns-thrrw.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-05T09:44:20.634Z","logger":"controllers.RayJob","msg":"user-provided submitter template is used; the first container is assumed to be the submitter","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"397d99c3-8cd3-4611-bdcb-797913d62af3"}
{"level":"info","ts":"2026-06-05T09:44:20.634Z","logger":"controllers.RayJob","msg":"No command is specified in the user-provided template. Default command is used","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"397d99c3-8cd3-4611-bdcb-797913d62af3","command":["if","!","ray","job","status","--address","http://counter-nl7mr-head-svc.test-ns-thrrw.svc.cluster.local:8265","counter-np762",">/dev/null","2>&1",";","then","ray","job","submit","--address","http://counter-nl7mr-head-svc.test-ns-thrrw.svc.cluster.local:8265","--no-wait","--runtime-env-json","\"{\\\"env_vars\\\":{\\\"counter_name\\\":\\\"test_counter\\\"}}\"","--submission-id","counter-np762","--","python /home/ray/jobs/counter.py",";","fi",";","ray","job","logs","--address","http://counter-nl7mr-head-svc.test-ns-thrrw.svc.cluster.local:8265","--follow","counter-np762"]}
{"level":"info","ts":"2026-06-05T09:44:20.642Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"397d99c3-8cd3-4611-bdcb-797913d62af3","Kubernetes Job":"counter"}
{"level":"info","ts":"2026-06-05T09:44:20.642Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"397d99c3-8cd3-4611-bdcb-797913d62af3","SubmissionMode":"K8sJobMode","RayCluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:20.642Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"397d99c3-8cd3-4611-bdcb-797913d62af3","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-np762","rayClusterName":"counter-nl7mr","jobDeploymentStatus":"Initializing","startTime":"2026-06-05T09:43:54Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-np762","rayClusterName":"counter-nl7mr","dashboardURL":"counter-nl7mr-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-05T09:43:54Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:44:20.642Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"397d99c3-8cd3-4611-bdcb-797913d62af3","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-05T09:44:20.649Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"2e75ebae-2a6b-4414-8b23-0c67e4990188","name":"counter-nl7mr-head"}
{"level":"info","ts":"2026-06-05T09:44:20.653Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"22b4354f-1d0e-4097-83d1-4e729bebb5c1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:44:20.653Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"22b4354f-1d0e-4097-83d1-4e729bebb5c1","name":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:20.653Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"22b4354f-1d0e-4097-83d1-4e729bebb5c1","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:44:20.653Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"22b4354f-1d0e-4097-83d1-4e729bebb5c1"}
{"level":"info","ts":"2026-06-05T09:44:20.653Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"22b4354f-1d0e-4097-83d1-4e729bebb5c1","head service name":"counter-nl7mr-head-svc","namespace":"test-ns-thrrw"}
{"level":"info","ts":"2026-06-05T09:44:20.653Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"22b4354f-1d0e-4097-83d1-4e729bebb5c1","head service URL":"counter-nl7mr-head-svc.test-ns-thrrw.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-05T09:44:20.653Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job for RayJob already exists","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"22b4354f-1d0e-4097-83d1-4e729bebb5c1","Kubernetes Job":"counter"}
{"level":"info","ts":"2026-06-05T09:44:20.653Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"22b4354f-1d0e-4097-83d1-4e729bebb5c1","SubmissionMode":"K8sJobMode","RayCluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:20.653Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"22b4354f-1d0e-4097-83d1-4e729bebb5c1","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-np762","rayClusterName":"counter-nl7mr","jobDeploymentStatus":"Initializing","startTime":"2026-06-05T09:43:54Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-np762","rayClusterName":"counter-nl7mr","dashboardURL":"counter-nl7mr-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-05T09:43:54Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:44:20.653Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"22b4354f-1d0e-4097-83d1-4e729bebb5c1","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-05T09:44:20.662Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"22b4354f-1d0e-4097-83d1-4e729bebb5c1","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-06-05T09:44:20.662Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"22b4354f-1d0e-4097-83d1-4e729bebb5c1"}
{"level":"error","ts":"2026-06-05T09:44:20.662Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"22b4354f-1d0e-4097-83d1-4e729bebb5c1","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:44:20.663Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"0adebc8c-5802-41ae-b647-bd551e0da738","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:44:20.663Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"0adebc8c-5802-41ae-b647-bd551e0da738","name":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:20.663Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"0adebc8c-5802-41ae-b647-bd551e0da738","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:44:20.663Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"0adebc8c-5802-41ae-b647-bd551e0da738"}
{"level":"info","ts":"2026-06-05T09:44:20.706Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"2e75ebae-2a6b-4414-8b23-0c67e4990188","name":"counter-nl7mr-workers"}
{"level":"info","ts":"2026-06-05T09:44:20.706Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"2e75ebae-2a6b-4414-8b23-0c67e4990188","cluster":"counter-nl7mr"}
{"level":"error","ts":"2026-06-05T09:44:20.713Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"0adebc8c-5802-41ae-b647-bd551e0da738","JobId":"counter-np762","error":"Job counter-np762 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:44:20.713Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"0adebc8c-5802-41ae-b647-bd551e0da738"}
{"level":"error","ts":"2026-06-05T09:44:20.713Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"0adebc8c-5802-41ae-b647-bd551e0da738","error":"Job counter-np762 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:44:20.713Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"057d4010-59bb-4391-bf02-75355f4189c1","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:44:20.713Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"057d4010-59bb-4391-bf02-75355f4189c1","name":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:20.713Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"057d4010-59bb-4391-bf02-75355f4189c1","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:44:20.713Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"057d4010-59bb-4391-bf02-75355f4189c1"}
{"level":"error","ts":"2026-06-05T09:44:20.718Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"057d4010-59bb-4391-bf02-75355f4189c1","JobId":"counter-np762","error":"Job counter-np762 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:44:20.718Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"057d4010-59bb-4391-bf02-75355f4189c1"}
{"level":"error","ts":"2026-06-05T09:44:20.718Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"057d4010-59bb-4391-bf02-75355f4189c1","error":"Job counter-np762 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:44:20.723Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"0005cc10-81d3-4c77-9f33-f1666610aab5","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:44:20.723Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"0005cc10-81d3-4c77-9f33-f1666610aab5","name":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:20.723Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"0005cc10-81d3-4c77-9f33-f1666610aab5","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:44:20.723Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"0005cc10-81d3-4c77-9f33-f1666610aab5"}
{"level":"error","ts":"2026-06-05T09:44:20.728Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"0005cc10-81d3-4c77-9f33-f1666610aab5","JobId":"counter-np762","error":"Job counter-np762 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:44:20.728Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"0005cc10-81d3-4c77-9f33-f1666610aab5"}
{"level":"error","ts":"2026-06-05T09:44:20.728Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"0005cc10-81d3-4c77-9f33-f1666610aab5","error":"Job counter-np762 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:44:20.768Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"9d9197a6-297d-41b6-80da-b3368f055ca9","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:44:20.768Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"9d9197a6-297d-41b6-80da-b3368f055ca9","name":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:20.768Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"9d9197a6-297d-41b6-80da-b3368f055ca9","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:44:20.768Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"9d9197a6-297d-41b6-80da-b3368f055ca9"}
{"level":"error","ts":"2026-06-05T09:44:20.808Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"9d9197a6-297d-41b6-80da-b3368f055ca9","JobId":"counter-np762","error":"Job counter-np762 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:44:20.808Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"9d9197a6-297d-41b6-80da-b3368f055ca9"}
{"level":"error","ts":"2026-06-05T09:44:20.808Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"9d9197a6-297d-41b6-80da-b3368f055ca9","error":"Job counter-np762 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:44:20.888Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"d751d742-c112-45b9-b743-9493fb70a7b1","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:44:20.888Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"d751d742-c112-45b9-b743-9493fb70a7b1","name":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:20.888Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"d751d742-c112-45b9-b743-9493fb70a7b1","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:44:20.888Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"d751d742-c112-45b9-b743-9493fb70a7b1"}
{"level":"error","ts":"2026-06-05T09:44:20.894Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"d751d742-c112-45b9-b743-9493fb70a7b1","JobId":"counter-np762","error":"Job counter-np762 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:44:20.894Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"d751d742-c112-45b9-b743-9493fb70a7b1"}
{"level":"error","ts":"2026-06-05T09:44:20.894Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"d751d742-c112-45b9-b743-9493fb70a7b1","error":"Job counter-np762 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:44:21.054Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"45d90671-20e3-4866-b0bc-5a669a2bae38","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:44:21.054Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"45d90671-20e3-4866-b0bc-5a669a2bae38","name":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:21.054Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"45d90671-20e3-4866-b0bc-5a669a2bae38","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:44:21.055Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"45d90671-20e3-4866-b0bc-5a669a2bae38"}
{"level":"error","ts":"2026-06-05T09:44:21.059Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"45d90671-20e3-4866-b0bc-5a669a2bae38","JobId":"counter-np762","error":"Job counter-np762 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:44:21.060Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"45d90671-20e3-4866-b0bc-5a669a2bae38"}
{"level":"error","ts":"2026-06-05T09:44:21.060Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"45d90671-20e3-4866-b0bc-5a669a2bae38","error":"Job counter-np762 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:44:21.381Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"b9e3fcc3-3561-45ec-8fd2-1f6aa7527fc0","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:44:21.381Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"b9e3fcc3-3561-45ec-8fd2-1f6aa7527fc0","name":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:21.381Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"b9e3fcc3-3561-45ec-8fd2-1f6aa7527fc0","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:44:21.381Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"b9e3fcc3-3561-45ec-8fd2-1f6aa7527fc0"}
{"level":"error","ts":"2026-06-05T09:44:21.386Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"b9e3fcc3-3561-45ec-8fd2-1f6aa7527fc0","JobId":"counter-np762","error":"Job counter-np762 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:44:21.386Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"b9e3fcc3-3561-45ec-8fd2-1f6aa7527fc0"}
{"level":"error","ts":"2026-06-05T09:44:21.386Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"b9e3fcc3-3561-45ec-8fd2-1f6aa7527fc0","error":"Job counter-np762 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:44:22.027Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"3f821ee5-9e59-4464-aaee-cb72428ea876","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:44:22.027Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"3f821ee5-9e59-4464-aaee-cb72428ea876","name":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:22.027Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"3f821ee5-9e59-4464-aaee-cb72428ea876","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:44:22.027Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"3f821ee5-9e59-4464-aaee-cb72428ea876"}
{"level":"error","ts":"2026-06-05T09:44:22.032Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"3f821ee5-9e59-4464-aaee-cb72428ea876","JobId":"counter-np762","error":"Job counter-np762 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:44:22.032Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"3f821ee5-9e59-4464-aaee-cb72428ea876"}
{"level":"error","ts":"2026-06-05T09:44:22.032Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"3f821ee5-9e59-4464-aaee-cb72428ea876","error":"Job counter-np762 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:44:22.634Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"07bd83dd-c617-47b9-bf74-687545a479ae"}
{"level":"info","ts":"2026-06-05T09:44:22.634Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"07bd83dd-c617-47b9-bf74-687545a479ae"}
{"level":"info","ts":"2026-06-05T09:44:22.634Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"07bd83dd-c617-47b9-bf74-687545a479ae","1 head service found":"counter-nl7mr-head-svc"}
{"level":"info","ts":"2026-06-05T09:44:22.634Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"07bd83dd-c617-47b9-bf74-687545a479ae","Found 1 head Pod":"counter-nl7mr-head-7xdm5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:44:22.634Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"07bd83dd-c617-47b9-bf74-687545a479ae","head Pod":"counter-nl7mr-head-7xdm5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-nl7mr-head-7xdm5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:44:22.634Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"07bd83dd-c617-47b9-bf74-687545a479ae","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:44:22.634Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"07bd83dd-c617-47b9-bf74-687545a479ae","worker Pod":"counter-nl7mr-small-group-worker-wpkdn","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-nl7mr-small-group-worker-wpkdn. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:44:22.634Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"07bd83dd-c617-47b9-bf74-687545a479ae","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:44:22.634Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"07bd83dd-c617-47b9-bf74-687545a479ae","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:44:22.634Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"07bd83dd-c617-47b9-bf74-687545a479ae","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:44:22.634Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"07bd83dd-c617-47b9-bf74-687545a479ae","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:44:22.634Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"07bd83dd-c617-47b9-bf74-687545a479ae","seconds":300}
{"level":"info","ts":"2026-06-05T09:44:22.674Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"a3b6a6a0-8f42-4503-94fb-31e87ce01002","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:44:22.674Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"a3b6a6a0-8f42-4503-94fb-31e87ce01002","name":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:22.674Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"a3b6a6a0-8f42-4503-94fb-31e87ce01002","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:44:22.674Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"a3b6a6a0-8f42-4503-94fb-31e87ce01002"}
{"level":"error","ts":"2026-06-05T09:44:22.680Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"a3b6a6a0-8f42-4503-94fb-31e87ce01002","JobId":"counter-np762","error":"Job counter-np762 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:44:22.680Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"a3b6a6a0-8f42-4503-94fb-31e87ce01002"}
{"level":"error","ts":"2026-06-05T09:44:22.680Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"a3b6a6a0-8f42-4503-94fb-31e87ce01002","error":"Job counter-np762 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:44:23.313Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"fc1b5705-e6b8-4dff-9ebb-24a829fd4a65","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:44:23.313Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"fc1b5705-e6b8-4dff-9ebb-24a829fd4a65","name":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:23.313Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"fc1b5705-e6b8-4dff-9ebb-24a829fd4a65","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:44:23.313Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"fc1b5705-e6b8-4dff-9ebb-24a829fd4a65"}
{"level":"error","ts":"2026-06-05T09:44:23.318Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"fc1b5705-e6b8-4dff-9ebb-24a829fd4a65","JobId":"counter-np762","error":"Job counter-np762 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:44:23.319Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"fc1b5705-e6b8-4dff-9ebb-24a829fd4a65"}
{"level":"error","ts":"2026-06-05T09:44:23.319Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"fc1b5705-e6b8-4dff-9ebb-24a829fd4a65","error":"Job counter-np762 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:44:25.113Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"775b3d58-c7a0-4e32-986f-e466c99f179b","rayCluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:25.113Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"775b3d58-c7a0-4e32-986f-e466c99f179b","pod":"counter-nl7mr-head-7xdm5","ip":"10.132.0.22"}
{"level":"info","ts":"2026-06-05T09:44:25.113Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"775b3d58-c7a0-4e32-986f-e466c99f179b","pod":"counter-nl7mr-small-group-worker-wpkdn","ip":"10.133.0.43"}
{"level":"info","ts":"2026-06-05T09:44:25.113Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"775b3d58-c7a0-4e32-986f-e466c99f179b","rayCluster":"counter-nl7mr","podIPs":["10.132.0.22","10.133.0.43"]}
{"level":"info","ts":"2026-06-05T09:44:28.439Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"9aa8f65e-5a62-4d17-9d68-f74ec53ef4d7","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:44:28.439Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"9aa8f65e-5a62-4d17-9d68-f74ec53ef4d7","name":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:28.439Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"9aa8f65e-5a62-4d17-9d68-f74ec53ef4d7","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:44:28.440Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"9aa8f65e-5a62-4d17-9d68-f74ec53ef4d7"}
{"level":"error","ts":"2026-06-05T09:44:28.445Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"9aa8f65e-5a62-4d17-9d68-f74ec53ef4d7","JobId":"counter-np762","error":"Job counter-np762 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:44:28.445Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"9aa8f65e-5a62-4d17-9d68-f74ec53ef4d7"}
{"level":"error","ts":"2026-06-05T09:44:28.445Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"9aa8f65e-5a62-4d17-9d68-f74ec53ef4d7","error":"Job counter-np762 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:44:33.607Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"d8a5e6ba-8753-4d1d-b63f-27b568d82dd4","namespace":"test-ns-t5s4r","clusterName":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:44:33.607Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-4qmws","namespace":"test-ns-t5s4r"},"namespace":"test-ns-t5s4r","name":"counter-4qmws","reconcileID":"d8a5e6ba-8753-4d1d-b63f-27b568d82dd4","namespace":"test-ns-t5s4r","clusterName":"counter-4qmws"}
{"level":"info","ts":"2026-06-05T09:44:36.552Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"00cfe9b8-9ad4-4054-8e6d-f3ee93724018","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:44:36.553Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"00cfe9b8-9ad4-4054-8e6d-f3ee93724018","pod":"raycluster-head-vjd7d","ip":"10.134.0.20"}
{"level":"info","ts":"2026-06-05T09:44:36.553Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"00cfe9b8-9ad4-4054-8e6d-f3ee93724018","pod":"raycluster-small-group-worker-j9pbw","ip":"10.133.0.35"}
{"level":"info","ts":"2026-06-05T09:44:36.553Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"00cfe9b8-9ad4-4054-8e6d-f3ee93724018","rayCluster":"raycluster","podIPs":["10.134.0.20","10.133.0.35"]}
{"level":"info","ts":"2026-06-05T09:44:38.687Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"431b15d7-60cd-48e8-b748-b4877e7b0ff7","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:44:38.687Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"431b15d7-60cd-48e8-b748-b4877e7b0ff7","name":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:38.687Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"431b15d7-60cd-48e8-b748-b4877e7b0ff7","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:44:38.687Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"431b15d7-60cd-48e8-b748-b4877e7b0ff7"}
{"level":"info","ts":"2026-06-05T09:44:38.695Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"431b15d7-60cd-48e8-b748-b4877e7b0ff7","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-np762","rayClusterName":"counter-nl7mr","dashboardURL":"counter-nl7mr-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-05T09:43:54Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:44:28Z","endTime":"2026-06-05T09:44:32Z"},"jobId":"counter-np762","rayClusterName":"counter-nl7mr","dashboardURL":"counter-nl7mr-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-05T09:43:54Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:44:20Z","stateTransitionTimes":{"ready":"2026-06-05T09:44:20Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.22","serviceIP":"10.132.0.22","podName":"counter-nl7mr-head-7xdm5","serviceName":"counter-nl7mr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:44:15Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:44:20Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:55Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:44:38.695Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"431b15d7-60cd-48e8-b748-b4877e7b0ff7","old JobStatus":"","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-05T09:44:38.709Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"45259310-1fdc-4089-8705-6899810aa72b","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:44:38.709Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"45259310-1fdc-4089-8705-6899810aa72b","name":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:38.709Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"45259310-1fdc-4089-8705-6899810aa72b","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:44:38.710Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"45259310-1fdc-4089-8705-6899810aa72b"}
{"level":"info","ts":"2026-06-05T09:44:38.712Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"45259310-1fdc-4089-8705-6899810aa72b","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:44:28Z","endTime":"2026-06-05T09:44:32Z"},"jobId":"counter-np762","rayClusterName":"counter-nl7mr","dashboardURL":"counter-nl7mr-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-05T09:43:54Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:44:20Z","stateTransitionTimes":{"ready":"2026-06-05T09:44:20Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.22","serviceIP":"10.132.0.22","podName":"counter-nl7mr-head-7xdm5","serviceName":"counter-nl7mr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:44:15Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:44:20Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:55Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:44:28Z","endTime":"2026-06-05T09:44:32Z"},"jobId":"counter-np762","rayClusterName":"counter-nl7mr","dashboardURL":"counter-nl7mr-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-05T09:43:54Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:44:20Z","stateTransitionTimes":{"ready":"2026-06-05T09:44:20Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.22","serviceIP":"10.132.0.22","podName":"counter-nl7mr-head-7xdm5","serviceName":"counter-nl7mr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:44:15Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:44:20Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:55Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:44:39.750Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"f11a5133-7f28-4ed7-914f-ecbda93ea8b5","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:44:39.750Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"f11a5133-7f28-4ed7-914f-ecbda93ea8b5","name":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:39.750Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"f11a5133-7f28-4ed7-914f-ecbda93ea8b5","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:44:39.750Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"f11a5133-7f28-4ed7-914f-ecbda93ea8b5"}
{"level":"info","ts":"2026-06-05T09:44:39.753Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"f11a5133-7f28-4ed7-914f-ecbda93ea8b5","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:44:28Z","endTime":"2026-06-05T09:44:32Z"},"jobId":"counter-np762","rayClusterName":"counter-nl7mr","dashboardURL":"counter-nl7mr-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-05T09:43:54Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:44:20Z","stateTransitionTimes":{"ready":"2026-06-05T09:44:20Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.22","serviceIP":"10.132.0.22","podName":"counter-nl7mr-head-7xdm5","serviceName":"counter-nl7mr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:44:15Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:44:20Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:55Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:44:28Z","endTime":"2026-06-05T09:44:32Z"},"jobId":"counter-np762","rayClusterName":"counter-nl7mr","dashboardURL":"counter-nl7mr-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-05T09:43:54Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:44:20Z","stateTransitionTimes":{"ready":"2026-06-05T09:44:20Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.22","serviceIP":"10.132.0.22","podName":"counter-nl7mr-head-7xdm5","serviceName":"counter-nl7mr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:44:15Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:44:20Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:55Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:44:40.762Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"ebb7ef2e-38e8-4971-8573-09dfd952a3d4","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:44:40.762Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"ebb7ef2e-38e8-4971-8573-09dfd952a3d4","name":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:40.762Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"ebb7ef2e-38e8-4971-8573-09dfd952a3d4","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:44:40.762Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"ebb7ef2e-38e8-4971-8573-09dfd952a3d4"}
{"level":"info","ts":"2026-06-05T09:44:40.765Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"ebb7ef2e-38e8-4971-8573-09dfd952a3d4","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:44:28Z","endTime":"2026-06-05T09:44:32Z"},"jobId":"counter-np762","rayClusterName":"counter-nl7mr","dashboardURL":"counter-nl7mr-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-05T09:43:54Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:44:20Z","stateTransitionTimes":{"ready":"2026-06-05T09:44:20Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.22","serviceIP":"10.132.0.22","podName":"counter-nl7mr-head-7xdm5","serviceName":"counter-nl7mr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:44:15Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:44:20Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:55Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:44:28Z","endTime":"2026-06-05T09:44:32Z"},"jobId":"counter-np762","rayClusterName":"counter-nl7mr","dashboardURL":"counter-nl7mr-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-05T09:43:54Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:44:20Z","stateTransitionTimes":{"ready":"2026-06-05T09:44:20Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.22","serviceIP":"10.132.0.22","podName":"counter-nl7mr-head-7xdm5","serviceName":"counter-nl7mr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:44:15Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:44:20Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:55Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:44:40.780Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"4a994f88-499b-46e1-be6c-aeac384d7cf1","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:44:40.780Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"4a994f88-499b-46e1-be6c-aeac384d7cf1","name":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:40.780Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"4a994f88-499b-46e1-be6c-aeac384d7cf1","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:44:40.780Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"4a994f88-499b-46e1-be6c-aeac384d7cf1"}
{"level":"info","ts":"2026-06-05T09:44:40.783Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"4a994f88-499b-46e1-be6c-aeac384d7cf1","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:44:28Z","endTime":"2026-06-05T09:44:32Z"},"jobId":"counter-np762","rayClusterName":"counter-nl7mr","dashboardURL":"counter-nl7mr-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-05T09:43:54Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:44:20Z","stateTransitionTimes":{"ready":"2026-06-05T09:44:20Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.22","serviceIP":"10.132.0.22","podName":"counter-nl7mr-head-7xdm5","serviceName":"counter-nl7mr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:44:15Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:44:20Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:55Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:44:28Z","endTime":"2026-06-05T09:44:32Z"},"jobId":"counter-np762","rayClusterName":"counter-nl7mr","dashboardURL":"counter-nl7mr-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Complete","message":"Job finished successfully.","startTime":"2026-06-05T09:43:54Z","succeeded":1,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:44:20Z","stateTransitionTimes":{"ready":"2026-06-05T09:44:20Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.22","serviceIP":"10.132.0.22","podName":"counter-nl7mr-head-7xdm5","serviceName":"counter-nl7mr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:44:15Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:44:20Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:43:54Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:43:55Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:44:40.784Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"4a994f88-499b-46e1-be6c-aeac384d7cf1","old JobStatus":"SUCCEEDED","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Complete"}
{"level":"info","ts":"2026-06-05T09:44:40.799Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"77fe714b-7024-4713-9009-4985fa10b0c9","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:44:40.799Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"77fe714b-7024-4713-9009-4985fa10b0c9","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-05 09:44:40 +0000 UTC","Now":"2026-06-05T09:44:40.799Z","ShutdownTime":"2026-06-05T09:44:40.000Z"}
{"level":"info","ts":"2026-06-05T09:44:40.799Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"77fe714b-7024-4713-9009-4985fa10b0c9","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-05T09:44:40.809Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob is deleted","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"77fe714b-7024-4713-9009-4985fa10b0c9","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:44:40.809Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"77fe714b-7024-4713-9009-4985fa10b0c9","isClusterDeleted":false}
{"level":"info","ts":"2026-06-05T09:44:40.809Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"77fe714b-7024-4713-9009-4985fa10b0c9","RayCluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:40.810Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"f92dd07d-b2c8-469f-a14c-cc6a9da4717b","rayCluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:40.810Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"e4a12101-0ece-4606-bef9-623eab0cff50","namespacedName":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:44:40.810Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"reconcileID":"91c0da4e-1d86-4b0a-8020-d247da06b664"}
{"level":"info","ts":"2026-06-05T09:44:40.810Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"e4a12101-0ece-4606-bef9-623eab0cff50","cluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:40.810Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"e4a12101-0ece-4606-bef9-623eab0cff50","cluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:40.810Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"66654dda-bd67-48fa-ba0a-d7a81f04ef46"}
{"level":"info","ts":"2026-06-05T09:44:40.810Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"f92dd07d-b2c8-469f-a14c-cc6a9da4717b","pod":"counter-nl7mr-head-7xdm5","ip":"10.132.0.22"}
{"level":"info","ts":"2026-06-05T09:44:40.810Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"f92dd07d-b2c8-469f-a14c-cc6a9da4717b","pod":"counter-nl7mr-small-group-worker-wpkdn","ip":"10.133.0.43"}
{"level":"info","ts":"2026-06-05T09:44:40.810Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"e4a12101-0ece-4606-bef9-623eab0cff50","namespace":"test-ns-thrrw","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-05T09:44:40.810Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"f92dd07d-b2c8-469f-a14c-cc6a9da4717b","rayCluster":"counter-nl7mr","podIPs":["10.132.0.22","10.133.0.43"]}
{"level":"info","ts":"2026-06-05T09:44:40.810Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"3479b36d-5e69-462c-8a25-d95208341cfb","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:44:40.810Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"3479b36d-5e69-462c-8a25-d95208341cfb","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-05 09:44:40 +0000 UTC","Now":"2026-06-05T09:44:40.810Z","ShutdownTime":"2026-06-05T09:44:40.000Z"}
{"level":"info","ts":"2026-06-05T09:44:40.810Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"3479b36d-5e69-462c-8a25-d95208341cfb","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-05T09:44:40.810Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"3479b36d-5e69-462c-8a25-d95208341cfb","RayCluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:40.810Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"3479b36d-5e69-462c-8a25-d95208341cfb","isClusterDeleted":false}
{"level":"info","ts":"2026-06-05T09:44:40.810Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"3479b36d-5e69-462c-8a25-d95208341cfb","RayCluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:40.816Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"e4a12101-0ece-4606-bef9-623eab0cff50","name":"kuberay-gateway-access","namespace":"test-ns-thrrw"}
{"level":"info","ts":"2026-06-05T09:44:40.822Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"e4a12101-0ece-4606-bef9-623eab0cff50","name":"test-ns-thrrw-counter-nl7mr","namespace":"opendatahub"}
{"level":"info","ts":"2026-06-05T09:44:40.828Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"e4a12101-0ece-4606-bef9-623eab0cff50","configMap":"kube-rbac-proxy-config-counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:40.833Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"e4a12101-0ece-4606-bef9-623eab0cff50","serviceAccount":"counter-nl7mr-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:44:40.842Z","logger":"raycluster-resource","msg":"default","name":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:40.842Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-nl7mr","namespace":"test-ns-thrrw"}
{"level":"info","ts":"2026-06-05T09:44:40.861Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"34382b9b-7c2e-4c06-b2ae-9215fc4cb9f3"}
{"level":"info","ts":"2026-06-05T09:44:40.861Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"a72f4d0d-1d8b-43cd-bab9-dd45c564994b","namespace":"test-ns-thrrw","clusterName":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:40.862Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"b37b6661-cb8a-4a25-82da-b5d1e020bfe8","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:44:40.862Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"b37b6661-cb8a-4a25-82da-b5d1e020bfe8","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-05 09:44:40 +0000 UTC","Now":"2026-06-05T09:44:40.862Z","ShutdownTime":"2026-06-05T09:44:40.000Z"}
{"level":"info","ts":"2026-06-05T09:44:40.862Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"b37b6661-cb8a-4a25-82da-b5d1e020bfe8","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-05T09:44:40.862Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"b37b6661-cb8a-4a25-82da-b5d1e020bfe8","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:44:40.862Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"b37b6661-cb8a-4a25-82da-b5d1e020bfe8","isClusterDeleted":true}
{"level":"info","ts":"2026-06-05T09:44:40.862Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"b37b6661-cb8a-4a25-82da-b5d1e020bfe8","RayCluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:40.862Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"e4a12101-0ece-4606-bef9-623eab0cff50","cluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:40.862Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"2fc4cddd-8d53-4290-a507-61cfb1e11d58","namespacedName":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:44:40.862Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"2fc4cddd-8d53-4290-a507-61cfb1e11d58"}
{"level":"info","ts":"2026-06-05T09:44:40.862Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"2fc4cddd-8d53-4290-a507-61cfb1e11d58","namespace":"test-ns-thrrw","authClusters":0}
{"level":"info","ts":"2026-06-05T09:44:40.862Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"2fc4cddd-8d53-4290-a507-61cfb1e11d58","namespace":"test-ns-thrrw"}
{"level":"info","ts":"2026-06-05T09:44:40.869Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"7ba30819-04b7-4321-8bae-f80ef7c098c2"}
{"level":"info","ts":"2026-06-05T09:44:40.869Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"e1d84742-abfd-498b-862c-4561f6739af0"}
{"level":"info","ts":"2026-06-05T09:44:40.870Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"d082154c-fe84-4365-a5f4-6aab79fd12dd","namespacedName":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:44:40.870Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"d082154c-fe84-4365-a5f4-6aab79fd12dd"}
{"level":"info","ts":"2026-06-05T09:44:40.870Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"d082154c-fe84-4365-a5f4-6aab79fd12dd","namespace":"test-ns-thrrw","authClusters":0}
{"level":"info","ts":"2026-06-05T09:44:40.870Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"d082154c-fe84-4365-a5f4-6aab79fd12dd","namespace":"test-ns-thrrw"}
{"level":"error","ts":"2026-06-05T09:44:40.872Z","msg":"Failed to delete SelfSigned issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"a72f4d0d-1d8b-43cd-bab9-dd45c564994b","issuer":"ray-selfsigned-issuer-counter-nl7mr","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-counter-nl7mr\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:254\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-05T09:44:40.878Z","msg":"Failed to delete head certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"a72f4d0d-1d8b-43cd-bab9-dd45c564994b","certificate":"ray-head-cert-counter-nl7mr","error":"certificates.cert-manager.io \"ray-head-cert-counter-nl7mr\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:261\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-05T09:44:40.906Z","msg":"Failed to delete worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"a72f4d0d-1d8b-43cd-bab9-dd45c564994b","certificate":"ray-worker-cert-counter-nl7mr","error":"certificates.cert-manager.io \"ray-worker-cert-counter-nl7mr\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:268\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:44:40.924Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"a72f4d0d-1d8b-43cd-bab9-dd45c564994b","errorCount":3}
{"level":"info","ts":"2026-06-05T09:44:40.924Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"a72f4d0d-1d8b-43cd-bab9-dd45c564994b"}
{"level":"error","ts":"2026-06-05T09:44:40.924Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"a72f4d0d-1d8b-43cd-bab9-dd45c564994b","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-counter-nl7mr\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:44:40.929Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"e9045457-cb10-4e80-8b6c-1e440787451c","namespace":"test-ns-thrrw","clusterName":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:40.929Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"counter-nl7mr","reconcileID":"e9045457-cb10-4e80-8b6c-1e440787451c","namespace":"test-ns-thrrw","clusterName":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:41.144Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"21da31ea-6eb7-4903-af01-f289c1fb310a","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:44:41.144Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"21da31ea-6eb7-4903-af01-f289c1fb310a","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-05 09:44:40 +0000 UTC","Now":"2026-06-05T09:44:41.144Z","ShutdownTime":"2026-06-05T09:44:40.000Z"}
{"level":"info","ts":"2026-06-05T09:44:41.144Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"21da31ea-6eb7-4903-af01-f289c1fb310a","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-05T09:44:41.144Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"21da31ea-6eb7-4903-af01-f289c1fb310a","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:44:41.144Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"21da31ea-6eb7-4903-af01-f289c1fb310a","isClusterDeleted":true}
{"level":"info","ts":"2026-06-05T09:44:41.144Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"21da31ea-6eb7-4903-af01-f289c1fb310a","RayCluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:44:41.710Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"e91ef72e-78a5-44b8-a5a7-6cec12860d12","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:44:41.710Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"e91ef72e-78a5-44b8-a5a7-6cec12860d12","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-05 09:44:40 +0000 UTC","Now":"2026-06-05T09:44:41.710Z","ShutdownTime":"2026-06-05T09:44:40.000Z"}
{"level":"info","ts":"2026-06-05T09:44:41.710Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"e91ef72e-78a5-44b8-a5a7-6cec12860d12","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-05T09:44:41.710Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"e91ef72e-78a5-44b8-a5a7-6cec12860d12","RayCluster":{"name":"counter-nl7mr","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:44:41.710Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"e91ef72e-78a5-44b8-a5a7-6cec12860d12","isClusterDeleted":true}
{"level":"info","ts":"2026-06-05T09:44:41.710Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"e91ef72e-78a5-44b8-a5a7-6cec12860d12","RayCluster":"counter-nl7mr"}
{"level":"info","ts":"2026-06-05T09:45:11.155Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"c4ea8b97-2371-4a3a-a416-0e4a52861532","DeletionTimestamp":"2026-06-05 09:45:11 +0000 UTC"}
{"level":"info","ts":"2026-06-05T09:45:11.155Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"c4ea8b97-2371-4a3a-a416-0e4a52861532","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-05T09:45:11.169Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"ee7e67cd-4a06-4f4f-abde-0082d0005ad4","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:45:11.169Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"ee7e67cd-4a06-4f4f-abde-0082d0005ad4","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-05T09:45:11.180Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"ee7e67cd-4a06-4f4f-abde-0082d0005ad4"}
{"level":"info","ts":"2026-06-05T09:45:11.180Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"ee7e67cd-4a06-4f4f-abde-0082d0005ad4","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-05T09:45:11.180Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"ee7e67cd-4a06-4f4f-abde-0082d0005ad4","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-qjbjr","rayClusterName":"fail-78fz8","jobDeploymentStatus":"Initializing","startTime":"2026-06-05T09:45:11Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:45:11.180Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"ee7e67cd-4a06-4f4f-abde-0082d0005ad4","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-05T09:45:11.191Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"e2cd0387-5c40-464e-b8fc-41b24e4e816a"}
{"level":"info","ts":"2026-06-05T09:45:11.191Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"4aa1e77c-fab3-40e5-9add-b4c39fb67e99","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:45:11.191Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"4aa1e77c-fab3-40e5-9add-b4c39fb67e99"}
{"level":"info","ts":"2026-06-05T09:45:11.191Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"4aa1e77c-fab3-40e5-9add-b4c39fb67e99","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-05T09:45:11.191Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"4aa1e77c-fab3-40e5-9add-b4c39fb67e99","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-ss5bq","rayClusterName":"fail-swh2x","jobDeploymentStatus":"Initializing","startTime":"2026-06-05T09:45:11Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:45:11.191Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"4aa1e77c-fab3-40e5-9add-b4c39fb67e99","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-05T09:45:11.197Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"4aa1e77c-fab3-40e5-9add-b4c39fb67e99","error":"Operation cannot be fulfilled on rayjobs.ray.io \"fail\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-06-05T09:45:11.197Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"4aa1e77c-fab3-40e5-9add-b4c39fb67e99"}
{"level":"error","ts":"2026-06-05T09:45:11.197Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"4aa1e77c-fab3-40e5-9add-b4c39fb67e99","error":"Operation cannot be fulfilled on rayjobs.ray.io \"fail\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:11.197Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"df5ad2ba-d2f5-4a11-846c-6649dd13672d"}
{"level":"info","ts":"2026-06-05T09:45:11.197Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"a30ebc4c-74a3-4298-be89-f3e9fd9f9fbc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:45:11.197Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"a30ebc4c-74a3-4298-be89-f3e9fd9f9fbc","name":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:11.197Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"a30ebc4c-74a3-4298-be89-f3e9fd9f9fbc","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:11.197Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"a30ebc4c-74a3-4298-be89-f3e9fd9f9fbc","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:11.203Z","logger":"raycluster-resource","msg":"default","name":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:11.203Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-78fz8","namespace":"test-ns-thrrw"}
{"level":"info","ts":"2026-06-05T09:45:11.219Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"a30ebc4c-74a3-4298-be89-f3e9fd9f9fbc","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:11.219Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"a30ebc4c-74a3-4298-be89-f3e9fd9f9fbc"}
{"level":"info","ts":"2026-06-05T09:45:11.219Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"a30ebc4c-74a3-4298-be89-f3e9fd9f9fbc","RayCluster":"fail-78fz8","State":""}
{"level":"info","ts":"2026-06-05T09:45:11.219Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"eb3e32aa-45e2-4eea-b27d-4685b9c69074","namespacedName":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:11.219Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"eb3e32aa-45e2-4eea-b27d-4685b9c69074","mode":"IntegratedOAuth","cluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:11.219Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"eb3e32aa-45e2-4eea-b27d-4685b9c69074","cluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:11.219Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"eb3e32aa-45e2-4eea-b27d-4685b9c69074","cluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:11.219Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"1e067dd5-4e11-487d-9018-fb9bc37c5bdf","rayCluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:11.219Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"f81a76d0-9938-408f-8d4a-de18e66b87f5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:45:11.219Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"f81a76d0-9938-408f-8d4a-de18e66b87f5","name":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:11.219Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"1e067dd5-4e11-487d-9018-fb9bc37c5bdf","rayCluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:11.219Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"b5c9a52d-4979-44d8-8a7b-8e52acd71509","cluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:11.219Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"f81a76d0-9938-408f-8d4a-de18e66b87f5","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:11.219Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"f28fc62d-7797-4393-ba55-ef91015e12eb"}
{"level":"info","ts":"2026-06-05T09:45:11.220Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"f28fc62d-7797-4393-ba55-ef91015e12eb"}
{"level":"info","ts":"2026-06-05T09:45:11.220Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"f81a76d0-9938-408f-8d4a-de18e66b87f5"}
{"level":"info","ts":"2026-06-05T09:45:11.220Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"f81a76d0-9938-408f-8d4a-de18e66b87f5","RayCluster":"fail-78fz8","State":""}
{"level":"info","ts":"2026-06-05T09:45:11.220Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"565e594f-f2de-483f-bb2e-087c8881c853","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:45:11.220Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"565e594f-f2de-483f-bb2e-087c8881c853","name":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:11.220Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"565e594f-f2de-483f-bb2e-087c8881c853","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:11.220Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"565e594f-f2de-483f-bb2e-087c8881c853"}
{"level":"info","ts":"2026-06-05T09:45:11.220Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"565e594f-f2de-483f-bb2e-087c8881c853","RayCluster":"fail-78fz8","State":""}
{"level":"info","ts":"2026-06-05T09:45:11.226Z","logger":"raycluster-resource","msg":"default","name":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:11.226Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-78fz8","namespace":"test-ns-thrrw"}
{"level":"info","ts":"2026-06-05T09:45:11.228Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"f28fc62d-7797-4393-ba55-ef91015e12eb","name":"fail-78fz8-head-svc"}
{"level":"info","ts":"2026-06-05T09:45:11.228Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"f28fc62d-7797-4393-ba55-ef91015e12eb","error":"head certificate secret ray-head-secret-fail-78fz8 not found"}
{"level":"error","ts":"2026-06-05T09:45:11.228Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"f28fc62d-7797-4393-ba55-ef91015e12eb","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-78fz8 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:11.228Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"f28fc62d-7797-4393-ba55-ef91015e12eb","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-78fz8 not found"}
{"level":"info","ts":"2026-06-05T09:45:11.228Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"f28fc62d-7797-4393-ba55-ef91015e12eb","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-78fz8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:11.228Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"f28fc62d-7797-4393-ba55-ef91015e12eb","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-78fz8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:11.228Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"f28fc62d-7797-4393-ba55-ef91015e12eb","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-78fz8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:11.228Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"f28fc62d-7797-4393-ba55-ef91015e12eb","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-05T09:45:11.228Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"f28fc62d-7797-4393-ba55-ef91015e12eb","name":"fail-78fz8","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:45:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-78fz8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:45:11.229Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"1e067dd5-4e11-487d-9018-fb9bc37c5bdf","rayCluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:11.229Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"1e067dd5-4e11-487d-9018-fb9bc37c5bdf","rayCluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:11.235Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"c44ffb9b-d001-43e8-a608-f26b648626bc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:45:11.235Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"c44ffb9b-d001-43e8-a608-f26b648626bc","name":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:11.235Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"c44ffb9b-d001-43e8-a608-f26b648626bc","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:11.235Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"c44ffb9b-d001-43e8-a608-f26b648626bc"}
{"level":"info","ts":"2026-06-05T09:45:11.235Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"c44ffb9b-d001-43e8-a608-f26b648626bc","RayCluster":"fail-78fz8","State":""}
{"level":"info","ts":"2026-06-05T09:45:11.306Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"b5c9a52d-4979-44d8-8a7b-8e52acd71509","cluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:11.306Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"1e067dd5-4e11-487d-9018-fb9bc37c5bdf"}
{"level":"info","ts":"2026-06-05T09:45:11.306Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"f28fc62d-7797-4393-ba55-ef91015e12eb","name":"fail-78fz8","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-78fz8\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-thrrw","name":"fail-78fz8"}}
{"level":"info","ts":"2026-06-05T09:45:11.306Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"f28fc62d-7797-4393-ba55-ef91015e12eb"}
{"level":"info","ts":"2026-06-05T09:45:11.306Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"1e067dd5-4e11-487d-9018-fb9bc37c5bdf","rayCluster":"fail-78fz8","secret":"ca-secret"}
{"level":"error","ts":"2026-06-05T09:45:11.306Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"f28fc62d-7797-4393-ba55-ef91015e12eb","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-78fz8 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:11.306Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"1e067dd5-4e11-487d-9018-fb9bc37c5bdf","rayCluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:11.306Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"eb3e32aa-45e2-4eea-b27d-4685b9c69074","name":"fail-78fz8-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:45:11.306Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"55b38149-60be-42d9-ae09-48aa2450a098"}
{"level":"info","ts":"2026-06-05T09:45:11.306Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"55b38149-60be-42d9-ae09-48aa2450a098"}
{"level":"info","ts":"2026-06-05T09:45:11.306Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"968612ff-2cff-4853-bbdc-2166c2f18c06","cluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:11.306Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"55b38149-60be-42d9-ae09-48aa2450a098","1 head service found":"fail-78fz8-head-svc"}
{"level":"info","ts":"2026-06-05T09:45:11.306Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"55b38149-60be-42d9-ae09-48aa2450a098","error":"head certificate secret ray-head-secret-fail-78fz8 not found"}
{"level":"error","ts":"2026-06-05T09:45:11.306Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"55b38149-60be-42d9-ae09-48aa2450a098","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-78fz8 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:11.306Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"55b38149-60be-42d9-ae09-48aa2450a098","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-78fz8 not found"}
{"level":"info","ts":"2026-06-05T09:45:11.306Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"55b38149-60be-42d9-ae09-48aa2450a098","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-78fz8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:11.306Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"55b38149-60be-42d9-ae09-48aa2450a098","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-78fz8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:11.306Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"55b38149-60be-42d9-ae09-48aa2450a098","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-78fz8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:11.306Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"55b38149-60be-42d9-ae09-48aa2450a098","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-05T09:45:11.306Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"55b38149-60be-42d9-ae09-48aa2450a098","name":"fail-78fz8","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:45:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-78fz8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:45:11.314Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"eb3e32aa-45e2-4eea-b27d-4685b9c69074","name":"kuberay-gateway-access","namespace":"test-ns-thrrw","operation":"created"}
{"level":"info","ts":"2026-06-05T09:45:11.317Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"55b38149-60be-42d9-ae09-48aa2450a098"}
{"level":"error","ts":"2026-06-05T09:45:11.317Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"55b38149-60be-42d9-ae09-48aa2450a098","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-78fz8 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:11.318Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"e542ed45-a25f-4546-9e6f-7e31ca227478"}
{"level":"info","ts":"2026-06-05T09:45:11.318Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"e542ed45-a25f-4546-9e6f-7e31ca227478"}
{"level":"info","ts":"2026-06-05T09:45:11.318Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"e542ed45-a25f-4546-9e6f-7e31ca227478","1 head service found":"fail-78fz8-head-svc"}
{"level":"info","ts":"2026-06-05T09:45:11.318Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"e542ed45-a25f-4546-9e6f-7e31ca227478","error":"head certificate secret ray-head-secret-fail-78fz8 not found"}
{"level":"error","ts":"2026-06-05T09:45:11.318Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"e542ed45-a25f-4546-9e6f-7e31ca227478","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-78fz8 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:11.318Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"e542ed45-a25f-4546-9e6f-7e31ca227478","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-78fz8 not found"}
{"level":"info","ts":"2026-06-05T09:45:11.318Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"e542ed45-a25f-4546-9e6f-7e31ca227478","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-78fz8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:11.318Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"e542ed45-a25f-4546-9e6f-7e31ca227478","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-78fz8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:11.318Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"e542ed45-a25f-4546-9e6f-7e31ca227478","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-78fz8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:11.318Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"e542ed45-a25f-4546-9e6f-7e31ca227478","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-05T09:45:11.318Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"e542ed45-a25f-4546-9e6f-7e31ca227478","name":"fail-78fz8","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:45:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-78fz8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:45:11.318Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"f7d62a23-8e58-4604-a35b-027741774358","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:45:11.318Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"f7d62a23-8e58-4604-a35b-027741774358","name":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:11.318Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"f7d62a23-8e58-4604-a35b-027741774358","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:11.318Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"f7d62a23-8e58-4604-a35b-027741774358"}
{"level":"info","ts":"2026-06-05T09:45:11.318Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"f7d62a23-8e58-4604-a35b-027741774358","RayCluster":"fail-78fz8","State":""}
{"level":"info","ts":"2026-06-05T09:45:11.319Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"1e067dd5-4e11-487d-9018-fb9bc37c5bdf","rayCluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:11.321Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"eb3e32aa-45e2-4eea-b27d-4685b9c69074","name":"test-ns-thrrw-fail-78fz8","operation":"created"}
{"level":"info","ts":"2026-06-05T09:45:11.324Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"968612ff-2cff-4853-bbdc-2166c2f18c06","name":"fail-78fz8-head"}
{"level":"info","ts":"2026-06-05T09:45:11.325Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"e542ed45-a25f-4546-9e6f-7e31ca227478","name":"fail-78fz8","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-78fz8\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-thrrw","name":"fail-78fz8"}}
{"level":"info","ts":"2026-06-05T09:45:11.325Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"e542ed45-a25f-4546-9e6f-7e31ca227478"}
{"level":"error","ts":"2026-06-05T09:45:11.325Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"e542ed45-a25f-4546-9e6f-7e31ca227478","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-78fz8 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:11.405Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"4dc3fb7a-b01d-4fbe-9237-878ed2c17758"}
{"level":"info","ts":"2026-06-05T09:45:11.405Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"4dc3fb7a-b01d-4fbe-9237-878ed2c17758"}
{"level":"info","ts":"2026-06-05T09:45:11.406Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"4dc3fb7a-b01d-4fbe-9237-878ed2c17758","1 head service found":"fail-78fz8-head-svc"}
{"level":"info","ts":"2026-06-05T09:45:11.406Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"4dc3fb7a-b01d-4fbe-9237-878ed2c17758","error":"head certificate secret ray-head-secret-fail-78fz8 not found"}
{"level":"error","ts":"2026-06-05T09:45:11.406Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"4dc3fb7a-b01d-4fbe-9237-878ed2c17758","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-78fz8 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:11.406Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"4dc3fb7a-b01d-4fbe-9237-878ed2c17758","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-78fz8 not found"}
{"level":"info","ts":"2026-06-05T09:45:11.406Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"4dc3fb7a-b01d-4fbe-9237-878ed2c17758","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-78fz8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:11.406Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"4dc3fb7a-b01d-4fbe-9237-878ed2c17758","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-78fz8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:11.406Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"4dc3fb7a-b01d-4fbe-9237-878ed2c17758","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-78fz8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:11.406Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"4dc3fb7a-b01d-4fbe-9237-878ed2c17758"}
{"level":"error","ts":"2026-06-05T09:45:11.406Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"4dc3fb7a-b01d-4fbe-9237-878ed2c17758","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-78fz8 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:11.406Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"1e067dd5-4e11-487d-9018-fb9bc37c5bdf"}
{"level":"info","ts":"2026-06-05T09:45:11.406Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"eb3e32aa-45e2-4eea-b27d-4685b9c69074","name":"kube-rbac-proxy-config-fail-78fz8","operation":"created"}
{"level":"info","ts":"2026-06-05T09:45:11.406Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"1e067dd5-4e11-487d-9018-fb9bc37c5bdf","rayCluster":"fail-78fz8","podIPs":[]}
{"level":"info","ts":"2026-06-05T09:45:11.411Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"968612ff-2cff-4853-bbdc-2166c2f18c06","name":"fail-78fz8-workers"}
{"level":"info","ts":"2026-06-05T09:45:11.411Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"968612ff-2cff-4853-bbdc-2166c2f18c06","cluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:11.411Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"b9a84cfa-37d5-4a4d-aed8-209b72950b55","cluster":"fail-78fz8"}
{"level":"error","ts":"2026-06-05T09:45:11.413Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"eb3e32aa-45e2-4eea-b27d-4685b9c69074","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-78fz8\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:11.413Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"eb3e32aa-45e2-4eea-b27d-4685b9c69074"}
{"level":"error","ts":"2026-06-05T09:45:11.413Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"eb3e32aa-45e2-4eea-b27d-4685b9c69074","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-78fz8\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:11.413Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"517d6c5e-990e-46c1-8ce2-d17009dd52fa","namespacedName":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:11.413Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"517d6c5e-990e-46c1-8ce2-d17009dd52fa","mode":"IntegratedOAuth","cluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:11.413Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"517d6c5e-990e-46c1-8ce2-d17009dd52fa","cluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:11.417Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"1e067dd5-4e11-487d-9018-fb9bc37c5bdf"}
{"level":"info","ts":"2026-06-05T09:45:11.418Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"1e067dd5-4e11-487d-9018-fb9bc37c5bdf","rayCluster":"fail-78fz8","podIPs":[]}
{"level":"info","ts":"2026-06-05T09:45:11.418Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"1e067dd5-4e11-487d-9018-fb9bc37c5bdf"}
{"level":"info","ts":"2026-06-05T09:45:11.418Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"9f50b711-8ef0-47fb-9d5b-6b8d75e558e3","rayCluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:11.418Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"9f50b711-8ef0-47fb-9d5b-6b8d75e558e3"}
{"level":"info","ts":"2026-06-05T09:45:11.425Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"b9a84cfa-37d5-4a4d-aed8-209b72950b55","name":"fail-78fz8-head"}
{"level":"info","ts":"2026-06-05T09:45:11.426Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"4cff071d-e3a5-4a72-ab61-815131379361","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:45:11.426Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"4cff071d-e3a5-4a72-ab61-815131379361","name":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:11.426Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"4cff071d-e3a5-4a72-ab61-815131379361","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:11.426Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"4cff071d-e3a5-4a72-ab61-815131379361"}
{"level":"info","ts":"2026-06-05T09:45:11.426Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"4cff071d-e3a5-4a72-ab61-815131379361","RayCluster":"fail-78fz8","State":""}
{"level":"info","ts":"2026-06-05T09:45:11.427Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"9fe9f3b9-5383-4e54-91fd-82d4bcf7ce54","rayCluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:11.427Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"9fe9f3b9-5383-4e54-91fd-82d4bcf7ce54"}
{"level":"info","ts":"2026-06-05T09:45:11.427Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"517d6c5e-990e-46c1-8ce2-d17009dd52fa","cluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:11.427Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"ccc702fe-8ab3-4e46-904d-7482bbc34e08","namespacedName":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:11.427Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"ccc702fe-8ab3-4e46-904d-7482bbc34e08","mode":"IntegratedOAuth","cluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:11.427Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"ccc702fe-8ab3-4e46-904d-7482bbc34e08","cluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:11.436Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"ccc702fe-8ab3-4e46-904d-7482bbc34e08","cluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:11.506Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"955f533a-a48e-4f06-8e7d-24cc47b1bc72"}
{"level":"info","ts":"2026-06-05T09:45:11.506Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"955f533a-a48e-4f06-8e7d-24cc47b1bc72"}
{"level":"info","ts":"2026-06-05T09:45:11.506Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"955f533a-a48e-4f06-8e7d-24cc47b1bc72","1 head service found":"fail-78fz8-head-svc"}
{"level":"info","ts":"2026-06-05T09:45:11.506Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"955f533a-a48e-4f06-8e7d-24cc47b1bc72","error":"head certificate secret ray-head-secret-fail-78fz8 not found"}
{"level":"info","ts":"2026-06-05T09:45:11.506Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"b9a84cfa-37d5-4a4d-aed8-209b72950b55","name":"fail-78fz8-workers"}
{"level":"error","ts":"2026-06-05T09:45:11.506Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"955f533a-a48e-4f06-8e7d-24cc47b1bc72","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-78fz8 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:11.506Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"955f533a-a48e-4f06-8e7d-24cc47b1bc72","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-78fz8 not found"}
{"level":"info","ts":"2026-06-05T09:45:11.506Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"b9a84cfa-37d5-4a4d-aed8-209b72950b55","cluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:11.506Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"955f533a-a48e-4f06-8e7d-24cc47b1bc72","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-78fz8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:11.506Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"955f533a-a48e-4f06-8e7d-24cc47b1bc72","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-78fz8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:11.506Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"f5d94374-9842-4eeb-ad23-ed68740e1d6a","cluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:11.506Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"955f533a-a48e-4f06-8e7d-24cc47b1bc72","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-78fz8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:11.506Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"955f533a-a48e-4f06-8e7d-24cc47b1bc72"}
{"level":"error","ts":"2026-06-05T09:45:11.506Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"955f533a-a48e-4f06-8e7d-24cc47b1bc72","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-78fz8 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:11.516Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"f5d94374-9842-4eeb-ad23-ed68740e1d6a","name":"fail-78fz8-head"}
{"level":"info","ts":"2026-06-05T09:45:11.527Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"f5d94374-9842-4eeb-ad23-ed68740e1d6a","name":"fail-78fz8-workers"}
{"level":"info","ts":"2026-06-05T09:45:11.527Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"f5d94374-9842-4eeb-ad23-ed68740e1d6a","cluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:11.587Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"643e266c-819b-4006-a01d-61b9c41e39e4"}
{"level":"info","ts":"2026-06-05T09:45:11.587Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"643e266c-819b-4006-a01d-61b9c41e39e4"}
{"level":"info","ts":"2026-06-05T09:45:11.587Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"643e266c-819b-4006-a01d-61b9c41e39e4","1 head service found":"fail-78fz8-head-svc"}
{"level":"info","ts":"2026-06-05T09:45:11.587Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"643e266c-819b-4006-a01d-61b9c41e39e4","error":"head certificate secret ray-head-secret-fail-78fz8 not found"}
{"level":"error","ts":"2026-06-05T09:45:11.587Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"643e266c-819b-4006-a01d-61b9c41e39e4","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-78fz8 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:11.587Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"643e266c-819b-4006-a01d-61b9c41e39e4","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-78fz8 not found"}
{"level":"info","ts":"2026-06-05T09:45:11.587Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"643e266c-819b-4006-a01d-61b9c41e39e4","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-78fz8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:11.587Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"643e266c-819b-4006-a01d-61b9c41e39e4","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-78fz8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:11.587Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"643e266c-819b-4006-a01d-61b9c41e39e4","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-78fz8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:11.587Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"643e266c-819b-4006-a01d-61b9c41e39e4"}
{"level":"error","ts":"2026-06-05T09:45:11.587Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"643e266c-819b-4006-a01d-61b9c41e39e4","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-78fz8 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:11.748Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"30d9f229-b761-457b-b8e8-3ca1d3a5f879"}
{"level":"info","ts":"2026-06-05T09:45:11.748Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"30d9f229-b761-457b-b8e8-3ca1d3a5f879"}
{"level":"info","ts":"2026-06-05T09:45:11.748Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"30d9f229-b761-457b-b8e8-3ca1d3a5f879","1 head service found":"fail-78fz8-head-svc"}
{"level":"info","ts":"2026-06-05T09:45:11.748Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"30d9f229-b761-457b-b8e8-3ca1d3a5f879","error":"head certificate secret ray-head-secret-fail-78fz8 not found"}
{"level":"error","ts":"2026-06-05T09:45:11.748Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"30d9f229-b761-457b-b8e8-3ca1d3a5f879","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-78fz8 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:11.748Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"30d9f229-b761-457b-b8e8-3ca1d3a5f879","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-78fz8 not found"}
{"level":"info","ts":"2026-06-05T09:45:11.748Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"30d9f229-b761-457b-b8e8-3ca1d3a5f879","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-78fz8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:11.748Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"30d9f229-b761-457b-b8e8-3ca1d3a5f879","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-78fz8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:11.748Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"30d9f229-b761-457b-b8e8-3ca1d3a5f879","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-78fz8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:11.748Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"30d9f229-b761-457b-b8e8-3ca1d3a5f879"}
{"level":"error","ts":"2026-06-05T09:45:11.748Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"30d9f229-b761-457b-b8e8-3ca1d3a5f879","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-78fz8 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:12.068Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"0a37c501-a21f-4870-82c9-a3a6bd793e64"}
{"level":"info","ts":"2026-06-05T09:45:12.068Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"0a37c501-a21f-4870-82c9-a3a6bd793e64"}
{"level":"info","ts":"2026-06-05T09:45:12.068Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"0a37c501-a21f-4870-82c9-a3a6bd793e64","1 head service found":"fail-78fz8-head-svc"}
{"level":"info","ts":"2026-06-05T09:45:12.068Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"0a37c501-a21f-4870-82c9-a3a6bd793e64","error":"head certificate secret ray-head-secret-fail-78fz8 not found"}
{"level":"error","ts":"2026-06-05T09:45:12.068Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"0a37c501-a21f-4870-82c9-a3a6bd793e64","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-78fz8 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:12.069Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"0a37c501-a21f-4870-82c9-a3a6bd793e64","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-78fz8 not found"}
{"level":"info","ts":"2026-06-05T09:45:12.069Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"0a37c501-a21f-4870-82c9-a3a6bd793e64","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-78fz8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:12.069Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"0a37c501-a21f-4870-82c9-a3a6bd793e64","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-78fz8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:12.069Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"0a37c501-a21f-4870-82c9-a3a6bd793e64","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-78fz8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:12.069Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"0a37c501-a21f-4870-82c9-a3a6bd793e64"}
{"level":"error","ts":"2026-06-05T09:45:12.069Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"0a37c501-a21f-4870-82c9-a3a6bd793e64","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-78fz8 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:12.710Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"cb5fa73f-0ed2-426e-9f37-c10b6ff14761"}
{"level":"info","ts":"2026-06-05T09:45:12.710Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"cb5fa73f-0ed2-426e-9f37-c10b6ff14761"}
{"level":"info","ts":"2026-06-05T09:45:12.710Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"cb5fa73f-0ed2-426e-9f37-c10b6ff14761","1 head service found":"fail-78fz8-head-svc"}
{"level":"info","ts":"2026-06-05T09:45:12.710Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"cb5fa73f-0ed2-426e-9f37-c10b6ff14761","error":"head certificate secret ray-head-secret-fail-78fz8 not found"}
{"level":"error","ts":"2026-06-05T09:45:12.710Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"cb5fa73f-0ed2-426e-9f37-c10b6ff14761","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-78fz8 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:12.710Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"cb5fa73f-0ed2-426e-9f37-c10b6ff14761","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-78fz8 not found"}
{"level":"info","ts":"2026-06-05T09:45:12.710Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"cb5fa73f-0ed2-426e-9f37-c10b6ff14761","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-78fz8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:12.710Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"cb5fa73f-0ed2-426e-9f37-c10b6ff14761","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-78fz8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:12.710Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"cb5fa73f-0ed2-426e-9f37-c10b6ff14761","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-78fz8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:12.710Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"cb5fa73f-0ed2-426e-9f37-c10b6ff14761"}
{"level":"error","ts":"2026-06-05T09:45:12.710Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"cb5fa73f-0ed2-426e-9f37-c10b6ff14761","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-78fz8 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:13.991Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"06067048-85bf-48c7-9af8-5fdf1b178279"}
{"level":"info","ts":"2026-06-05T09:45:13.991Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"06067048-85bf-48c7-9af8-5fdf1b178279"}
{"level":"info","ts":"2026-06-05T09:45:13.991Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"06067048-85bf-48c7-9af8-5fdf1b178279","1 head service found":"fail-78fz8-head-svc"}
{"level":"info","ts":"2026-06-05T09:45:13.991Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"06067048-85bf-48c7-9af8-5fdf1b178279"}
{"level":"info","ts":"2026-06-05T09:45:13.991Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"06067048-85bf-48c7-9af8-5fdf1b178279","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:45:13.991Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"06067048-85bf-48c7-9af8-5fdf1b178279"}
{"level":"info","ts":"2026-06-05T09:45:13.991Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"06067048-85bf-48c7-9af8-5fdf1b178279","mode":"IntegratedOAuth","cluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:13.991Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"06067048-85bf-48c7-9af8-5fdf1b178279","cluster":"fail-78fz8","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:45:13.991Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"06067048-85bf-48c7-9af8-5fdf1b178279","cluster":"fail-78fz8","authType":"IntegratedOAuth","serviceAccount":"fail-78fz8-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-05T09:45:13.991Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"06067048-85bf-48c7-9af8-5fdf1b178279","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"fail-78fz8","ray.io/group":"headgroup","ray.io/identifier":"fail-78fz8-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-05T09:45:13.991Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"06067048-85bf-48c7-9af8-5fdf1b178279","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"fail-78fz8-head-svc.test-ns-thrrw.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-05T09:45:13.991Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"06067048-85bf-48c7-9af8-5fdf1b178279","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-78fz8-head-svc.test-ns-thrrw.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-05T09:45:13.991Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"06067048-85bf-48c7-9af8-5fdf1b178279","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-78fz8-head-svc.test-ns-thrrw.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-05T09:45:13.991Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"06067048-85bf-48c7-9af8-5fdf1b178279","enabled":true}
{"level":"info","ts":"2026-06-05T09:45:14.029Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"06067048-85bf-48c7-9af8-5fdf1b178279","name":"fail-78fz8-head-7jx8g"}
{"level":"info","ts":"2026-06-05T09:45:14.029Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"06067048-85bf-48c7-9af8-5fdf1b178279","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:45:14.029Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"06067048-85bf-48c7-9af8-5fdf1b178279","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:45:14.029Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"06067048-85bf-48c7-9af8-5fdf1b178279","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-05T09:45:14.029Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"06067048-85bf-48c7-9af8-5fdf1b178279","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-05T09:45:14.029Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"06067048-85bf-48c7-9af8-5fdf1b178279","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-05T09:45:14.029Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"06067048-85bf-48c7-9af8-5fdf1b178279"}
{"level":"info","ts":"2026-06-05T09:45:14.029Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"06067048-85bf-48c7-9af8-5fdf1b178279","nodeType":"worker","rayStartParams":{"address":"fail-78fz8-head-svc.test-ns-thrrw.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-05T09:45:14.029Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"06067048-85bf-48c7-9af8-5fdf1b178279","rayStartCmd":"ray start  --address=fail-78fz8-head-svc.test-ns-thrrw.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:45:14.029Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"06067048-85bf-48c7-9af8-5fdf1b178279","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-78fz8-head-svc.test-ns-thrrw.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:45:14.029Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"06067048-85bf-48c7-9af8-5fdf1b178279","enabled":true}
{"level":"info","ts":"2026-06-05T09:45:14.068Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"06067048-85bf-48c7-9af8-5fdf1b178279","name":"fail-78fz8-small-group-worker-l8sb8"}
{"level":"info","ts":"2026-06-05T09:45:14.068Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"06067048-85bf-48c7-9af8-5fdf1b178279","podName":"fail-78fz8-head-7jx8g","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:45:14.068Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"06067048-85bf-48c7-9af8-5fdf1b178279","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"fail-78fz8-head-svc"},"newHead":{"podName":"fail-78fz8-head-7jx8g","serviceName":"fail-78fz8-head-svc"}}
{"level":"info","ts":"2026-06-05T09:45:14.068Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"06067048-85bf-48c7-9af8-5fdf1b178279","name":"fail-78fz8","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:45:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-78fz8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:45:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-78fz8-head-7jx8g","serviceName":"fail-78fz8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:45:14.107Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5b859c28-60fb-4825-ad8d-e2e734f840f4"}
{"level":"info","ts":"2026-06-05T09:45:14.107Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5b859c28-60fb-4825-ad8d-e2e734f840f4"}
{"level":"info","ts":"2026-06-05T09:45:14.107Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5b859c28-60fb-4825-ad8d-e2e734f840f4","1 head service found":"fail-78fz8-head-svc"}
{"level":"info","ts":"2026-06-05T09:45:14.107Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5b859c28-60fb-4825-ad8d-e2e734f840f4","Found 1 head Pod":"fail-78fz8-head-7jx8g","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:45:14.107Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5b859c28-60fb-4825-ad8d-e2e734f840f4","head Pod":"fail-78fz8-head-7jx8g","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-78fz8-head-7jx8g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:45:14.107Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5b859c28-60fb-4825-ad8d-e2e734f840f4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:45:14.107Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5b859c28-60fb-4825-ad8d-e2e734f840f4","worker Pod":"fail-78fz8-small-group-worker-l8sb8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-78fz8-small-group-worker-l8sb8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:45:14.107Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5b859c28-60fb-4825-ad8d-e2e734f840f4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:45:14.107Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5b859c28-60fb-4825-ad8d-e2e734f840f4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:45:14.107Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5b859c28-60fb-4825-ad8d-e2e734f840f4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:45:14.108Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5b859c28-60fb-4825-ad8d-e2e734f840f4","podName":"fail-78fz8-head-7jx8g","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:45:14.108Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5b859c28-60fb-4825-ad8d-e2e734f840f4","podName":"fail-78fz8-head-7jx8g","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:45:14.108Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5b859c28-60fb-4825-ad8d-e2e734f840f4","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"fail-78fz8-head-svc"},"newHead":{"podName":"fail-78fz8-head-7jx8g","serviceName":"fail-78fz8-head-svc"}}
{"level":"info","ts":"2026-06-05T09:45:14.108Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5b859c28-60fb-4825-ad8d-e2e734f840f4","name":"fail-78fz8","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:45:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-78fz8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:45:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-78fz8-head-7jx8g","serviceName":"fail-78fz8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:45:14.108Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"85e49f12-17b7-499c-97c9-cb69a76d16ab","rayCluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:14.109Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"85e49f12-17b7-499c-97c9-cb69a76d16ab","rayCluster":"fail-78fz8","podIPs":[]}
{"level":"info","ts":"2026-06-05T09:45:14.109Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"290d4f96-8511-4784-a731-fe6c7abf29f3","cluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:14.109Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"b9632c62-8e95-442e-9de0-99c242435afd","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:45:14.109Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"b9632c62-8e95-442e-9de0-99c242435afd","name":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:14.109Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"b9632c62-8e95-442e-9de0-99c242435afd","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:14.110Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"b9632c62-8e95-442e-9de0-99c242435afd"}
{"level":"info","ts":"2026-06-05T09:45:14.110Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"b9632c62-8e95-442e-9de0-99c242435afd","RayCluster":"fail-78fz8","State":""}
{"level":"info","ts":"2026-06-05T09:45:14.205Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"counter","namespace":"test-ns-thrrw"},"reconcileID":"1dbb00f4-2732-4f1e-8984-d80f9b8edd76"}
{"level":"info","ts":"2026-06-05T09:45:14.206Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5b859c28-60fb-4825-ad8d-e2e734f840f4","name":"fail-78fz8","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-78fz8\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-thrrw","name":"fail-78fz8"}}
{"level":"info","ts":"2026-06-05T09:45:14.206Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5b859c28-60fb-4825-ad8d-e2e734f840f4"}
{"level":"error","ts":"2026-06-05T09:45:14.206Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5b859c28-60fb-4825-ad8d-e2e734f840f4","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-78fz8\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:14.206Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"a930c9b3-c995-4577-8bde-baac00c6c88e"}
{"level":"info","ts":"2026-06-05T09:45:14.206Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"a930c9b3-c995-4577-8bde-baac00c6c88e"}
{"level":"info","ts":"2026-06-05T09:45:14.206Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"a930c9b3-c995-4577-8bde-baac00c6c88e","1 head service found":"fail-78fz8-head-svc"}
{"level":"info","ts":"2026-06-05T09:45:14.207Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"a930c9b3-c995-4577-8bde-baac00c6c88e","Found 1 head Pod":"fail-78fz8-head-7jx8g","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:45:14.207Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"a930c9b3-c995-4577-8bde-baac00c6c88e","head Pod":"fail-78fz8-head-7jx8g","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-78fz8-head-7jx8g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:45:14.207Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"a930c9b3-c995-4577-8bde-baac00c6c88e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:45:14.207Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"a930c9b3-c995-4577-8bde-baac00c6c88e","worker Pod":"fail-78fz8-small-group-worker-l8sb8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-78fz8-small-group-worker-l8sb8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:45:14.207Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"a930c9b3-c995-4577-8bde-baac00c6c88e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:45:14.207Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"a930c9b3-c995-4577-8bde-baac00c6c88e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:45:14.207Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"a930c9b3-c995-4577-8bde-baac00c6c88e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:45:14.207Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"a930c9b3-c995-4577-8bde-baac00c6c88e","podName":"fail-78fz8-head-7jx8g","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:45:14.207Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"a930c9b3-c995-4577-8bde-baac00c6c88e","podName":"fail-78fz8-head-7jx8g","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:45:14.207Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"a930c9b3-c995-4577-8bde-baac00c6c88e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:45:14.207Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"a930c9b3-c995-4577-8bde-baac00c6c88e","seconds":300}
{"level":"info","ts":"2026-06-05T09:45:14.207Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"154fdd8c-12af-415d-ba36-a930314b8917"}
{"level":"info","ts":"2026-06-05T09:45:14.207Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"154fdd8c-12af-415d-ba36-a930314b8917"}
{"level":"info","ts":"2026-06-05T09:45:14.207Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"154fdd8c-12af-415d-ba36-a930314b8917","1 head service found":"fail-78fz8-head-svc"}
{"level":"info","ts":"2026-06-05T09:45:14.207Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"154fdd8c-12af-415d-ba36-a930314b8917","Found 1 head Pod":"fail-78fz8-head-7jx8g","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:45:14.207Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"154fdd8c-12af-415d-ba36-a930314b8917","head Pod":"fail-78fz8-head-7jx8g","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-78fz8-head-7jx8g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:45:14.207Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"154fdd8c-12af-415d-ba36-a930314b8917","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:45:14.207Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"154fdd8c-12af-415d-ba36-a930314b8917","worker Pod":"fail-78fz8-small-group-worker-l8sb8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-78fz8-small-group-worker-l8sb8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:45:14.208Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"154fdd8c-12af-415d-ba36-a930314b8917","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:45:14.208Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"154fdd8c-12af-415d-ba36-a930314b8917","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:45:14.208Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"154fdd8c-12af-415d-ba36-a930314b8917","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:45:14.208Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"154fdd8c-12af-415d-ba36-a930314b8917","podName":"fail-78fz8-small-group-worker-l8sb8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:45:14.208Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"154fdd8c-12af-415d-ba36-a930314b8917","podName":"fail-78fz8-small-group-worker-l8sb8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:45:14.208Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"154fdd8c-12af-415d-ba36-a930314b8917","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-05T09:45:14.208Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"154fdd8c-12af-415d-ba36-a930314b8917","name":"fail-78fz8","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:45:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-78fz8-head-7jx8g","serviceName":"fail-78fz8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:45:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-78fz8-head-7jx8g","serviceName":"fail-78fz8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:45:14.306Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"2f94a813-f0d4-40e2-98aa-86d022ba90fc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:45:14.306Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"2f94a813-f0d4-40e2-98aa-86d022ba90fc","name":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:14.306Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"2f94a813-f0d4-40e2-98aa-86d022ba90fc","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:14.306Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"2f94a813-f0d4-40e2-98aa-86d022ba90fc"}
{"level":"info","ts":"2026-06-05T09:45:14.306Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"2f94a813-f0d4-40e2-98aa-86d022ba90fc","RayCluster":"fail-78fz8","State":""}
{"level":"info","ts":"2026-06-05T09:45:14.311Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"290d4f96-8511-4784-a731-fe6c7abf29f3","name":"fail-78fz8-head"}
{"level":"info","ts":"2026-06-05T09:45:14.312Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"cb56cb46-b201-49bb-b818-5fcf5892fb22"}
{"level":"info","ts":"2026-06-05T09:45:14.312Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"cb56cb46-b201-49bb-b818-5fcf5892fb22"}
{"level":"info","ts":"2026-06-05T09:45:14.313Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"cb56cb46-b201-49bb-b818-5fcf5892fb22","1 head service found":"fail-78fz8-head-svc"}
{"level":"info","ts":"2026-06-05T09:45:14.313Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"cb56cb46-b201-49bb-b818-5fcf5892fb22","Found 1 head Pod":"fail-78fz8-head-7jx8g","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:45:14.313Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"cb56cb46-b201-49bb-b818-5fcf5892fb22","head Pod":"fail-78fz8-head-7jx8g","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-78fz8-head-7jx8g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:45:14.313Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"cb56cb46-b201-49bb-b818-5fcf5892fb22","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:45:14.313Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"cb56cb46-b201-49bb-b818-5fcf5892fb22","worker Pod":"fail-78fz8-small-group-worker-l8sb8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-78fz8-small-group-worker-l8sb8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:45:14.313Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"cb56cb46-b201-49bb-b818-5fcf5892fb22","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:45:14.313Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"cb56cb46-b201-49bb-b818-5fcf5892fb22","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:45:14.313Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"cb56cb46-b201-49bb-b818-5fcf5892fb22","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:45:14.313Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"cb56cb46-b201-49bb-b818-5fcf5892fb22","podName":"fail-78fz8-head-7jx8g","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:45:14.313Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"5058e80f-6a71-4574-8956-882d099847cb","rayCluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:14.314Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"cb56cb46-b201-49bb-b818-5fcf5892fb22","podName":"fail-78fz8-head-7jx8g","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:45:14.314Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"5058e80f-6a71-4574-8956-882d099847cb","rayCluster":"fail-78fz8","podIPs":[]}
{"level":"info","ts":"2026-06-05T09:45:14.314Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"cb56cb46-b201-49bb-b818-5fcf5892fb22","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-05T09:45:14.314Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"cb56cb46-b201-49bb-b818-5fcf5892fb22","name":"fail-78fz8","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:45:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-78fz8-head-7jx8g","serviceName":"fail-78fz8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:45:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-78fz8-head-7jx8g","serviceName":"fail-78fz8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:45:14.314Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"ea217d5b-ef35-4c35-bae6-9c831624c7b2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:45:14.314Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"ea217d5b-ef35-4c35-bae6-9c831624c7b2","name":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:14.314Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"ea217d5b-ef35-4c35-bae6-9c831624c7b2","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:14.315Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"ea217d5b-ef35-4c35-bae6-9c831624c7b2"}
{"level":"info","ts":"2026-06-05T09:45:14.315Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"ea217d5b-ef35-4c35-bae6-9c831624c7b2","RayCluster":"fail-78fz8","State":""}
{"level":"info","ts":"2026-06-05T09:45:14.406Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"cb56cb46-b201-49bb-b818-5fcf5892fb22","name":"fail-78fz8","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-78fz8\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-thrrw","name":"fail-78fz8"}}
{"level":"info","ts":"2026-06-05T09:45:14.406Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"cb56cb46-b201-49bb-b818-5fcf5892fb22"}
{"level":"error","ts":"2026-06-05T09:45:14.406Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"cb56cb46-b201-49bb-b818-5fcf5892fb22","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-78fz8\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:14.411Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5a60f1bf-6efe-4383-80a2-976152779ad5"}
{"level":"info","ts":"2026-06-05T09:45:14.411Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5a60f1bf-6efe-4383-80a2-976152779ad5"}
{"level":"info","ts":"2026-06-05T09:45:14.411Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5a60f1bf-6efe-4383-80a2-976152779ad5","1 head service found":"fail-78fz8-head-svc"}
{"level":"info","ts":"2026-06-05T09:45:14.411Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5a60f1bf-6efe-4383-80a2-976152779ad5","Found 1 head Pod":"fail-78fz8-head-7jx8g","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:45:14.411Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5a60f1bf-6efe-4383-80a2-976152779ad5","head Pod":"fail-78fz8-head-7jx8g","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-78fz8-head-7jx8g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:45:14.411Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5a60f1bf-6efe-4383-80a2-976152779ad5","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:45:14.412Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5a60f1bf-6efe-4383-80a2-976152779ad5","worker Pod":"fail-78fz8-small-group-worker-l8sb8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-78fz8-small-group-worker-l8sb8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:45:14.412Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5a60f1bf-6efe-4383-80a2-976152779ad5","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:45:14.412Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5a60f1bf-6efe-4383-80a2-976152779ad5","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:45:14.412Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5a60f1bf-6efe-4383-80a2-976152779ad5","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:45:14.412Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5a60f1bf-6efe-4383-80a2-976152779ad5","podName":"fail-78fz8-head-7jx8g","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:45:14.412Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5a60f1bf-6efe-4383-80a2-976152779ad5","podName":"fail-78fz8-head-7jx8g","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:45:14.412Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5a60f1bf-6efe-4383-80a2-976152779ad5","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:45:14.412Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5a60f1bf-6efe-4383-80a2-976152779ad5","seconds":300}
{"level":"info","ts":"2026-06-05T09:45:14.418Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"290d4f96-8511-4784-a731-fe6c7abf29f3","name":"fail-78fz8-workers"}
{"level":"info","ts":"2026-06-05T09:45:14.418Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"290d4f96-8511-4784-a731-fe6c7abf29f3","cluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:14.418Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"2e8a91ee-73db-48db-82c3-3646473d851e","cluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:14.437Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"2e8a91ee-73db-48db-82c3-3646473d851e","name":"fail-78fz8-head"}
{"level":"info","ts":"2026-06-05T09:45:14.448Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"2e8a91ee-73db-48db-82c3-3646473d851e","name":"fail-78fz8-workers"}
{"level":"info","ts":"2026-06-05T09:45:14.448Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"2e8a91ee-73db-48db-82c3-3646473d851e","cluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:14.487Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"1f271370-dbe0-44a5-b47c-b1a3393e181b"}
{"level":"info","ts":"2026-06-05T09:45:14.487Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"1f271370-dbe0-44a5-b47c-b1a3393e181b"}
{"level":"info","ts":"2026-06-05T09:45:14.487Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"1f271370-dbe0-44a5-b47c-b1a3393e181b","1 head service found":"fail-78fz8-head-svc"}
{"level":"info","ts":"2026-06-05T09:45:14.487Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"1f271370-dbe0-44a5-b47c-b1a3393e181b","Found 1 head Pod":"fail-78fz8-head-7jx8g","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:45:14.487Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"1f271370-dbe0-44a5-b47c-b1a3393e181b","head Pod":"fail-78fz8-head-7jx8g","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-78fz8-head-7jx8g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:45:14.487Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"1f271370-dbe0-44a5-b47c-b1a3393e181b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:45:14.487Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"1f271370-dbe0-44a5-b47c-b1a3393e181b","worker Pod":"fail-78fz8-small-group-worker-l8sb8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-78fz8-small-group-worker-l8sb8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:45:14.487Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"1f271370-dbe0-44a5-b47c-b1a3393e181b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:45:14.487Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"1f271370-dbe0-44a5-b47c-b1a3393e181b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:45:14.487Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"1f271370-dbe0-44a5-b47c-b1a3393e181b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:45:14.487Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"1f271370-dbe0-44a5-b47c-b1a3393e181b","podName":"fail-78fz8-head-7jx8g","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:45:14.487Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"1f271370-dbe0-44a5-b47c-b1a3393e181b","podName":"fail-78fz8-head-7jx8g","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:45:14.487Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"1f271370-dbe0-44a5-b47c-b1a3393e181b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:45:14.487Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"1f271370-dbe0-44a5-b47c-b1a3393e181b","seconds":300}
{"level":"info","ts":"2026-06-05T09:45:14.555Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"f948e80e-e785-43ca-91c7-989e30b50f82"}
{"level":"info","ts":"2026-06-05T09:45:14.555Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"f948e80e-e785-43ca-91c7-989e30b50f82"}
{"level":"info","ts":"2026-06-05T09:45:14.556Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"f948e80e-e785-43ca-91c7-989e30b50f82","1 head service found":"fail-78fz8-head-svc"}
{"level":"info","ts":"2026-06-05T09:45:14.556Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"f948e80e-e785-43ca-91c7-989e30b50f82","Found 1 head Pod":"fail-78fz8-head-7jx8g","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:45:14.556Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"f948e80e-e785-43ca-91c7-989e30b50f82","head Pod":"fail-78fz8-head-7jx8g","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-78fz8-head-7jx8g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:45:14.556Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"f948e80e-e785-43ca-91c7-989e30b50f82","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:45:14.556Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"f948e80e-e785-43ca-91c7-989e30b50f82","worker Pod":"fail-78fz8-small-group-worker-l8sb8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-78fz8-small-group-worker-l8sb8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:45:14.556Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"f948e80e-e785-43ca-91c7-989e30b50f82","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:45:14.556Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"f948e80e-e785-43ca-91c7-989e30b50f82","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:45:14.556Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"f948e80e-e785-43ca-91c7-989e30b50f82","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:45:14.556Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"f948e80e-e785-43ca-91c7-989e30b50f82","podName":"fail-78fz8-head-7jx8g","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:45:14.556Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"f948e80e-e785-43ca-91c7-989e30b50f82","podName":"fail-78fz8-head-7jx8g","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:45:14.556Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"f948e80e-e785-43ca-91c7-989e30b50f82","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:45:14.556Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"f948e80e-e785-43ca-91c7-989e30b50f82","seconds":300}
{"level":"info","ts":"2026-06-05T09:45:14.923Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"2cde1c53-495f-40a7-a041-c583b5b65770"}
{"level":"info","ts":"2026-06-05T09:45:14.923Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"2cde1c53-495f-40a7-a041-c583b5b65770"}
{"level":"info","ts":"2026-06-05T09:45:14.923Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"2cde1c53-495f-40a7-a041-c583b5b65770","1 head service found":"fail-78fz8-head-svc"}
{"level":"info","ts":"2026-06-05T09:45:14.923Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"2cde1c53-495f-40a7-a041-c583b5b65770","Found 1 head Pod":"fail-78fz8-head-7jx8g","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:45:14.923Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"2cde1c53-495f-40a7-a041-c583b5b65770","head Pod":"fail-78fz8-head-7jx8g","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-78fz8-head-7jx8g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:45:14.923Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"2cde1c53-495f-40a7-a041-c583b5b65770","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:45:14.924Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"2cde1c53-495f-40a7-a041-c583b5b65770","worker Pod":"fail-78fz8-small-group-worker-l8sb8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-78fz8-small-group-worker-l8sb8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:45:14.924Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"2cde1c53-495f-40a7-a041-c583b5b65770","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:45:14.924Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"2cde1c53-495f-40a7-a041-c583b5b65770","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:45:14.924Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"2cde1c53-495f-40a7-a041-c583b5b65770","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:45:14.924Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"2cde1c53-495f-40a7-a041-c583b5b65770","podName":"fail-78fz8-small-group-worker-l8sb8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:45:14.924Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"2cde1c53-495f-40a7-a041-c583b5b65770","podName":"fail-78fz8-small-group-worker-l8sb8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:45:14.924Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"2cde1c53-495f-40a7-a041-c583b5b65770","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:45:14.924Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"2cde1c53-495f-40a7-a041-c583b5b65770","seconds":300}
{"level":"info","ts":"2026-06-05T09:45:15.032Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5603511a-d427-4075-aaec-04f2a558c64d"}
{"level":"info","ts":"2026-06-05T09:45:15.032Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5603511a-d427-4075-aaec-04f2a558c64d"}
{"level":"info","ts":"2026-06-05T09:45:15.032Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5603511a-d427-4075-aaec-04f2a558c64d","1 head service found":"fail-78fz8-head-svc"}
{"level":"info","ts":"2026-06-05T09:45:15.032Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5603511a-d427-4075-aaec-04f2a558c64d","Found 1 head Pod":"fail-78fz8-head-7jx8g","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:45:15.032Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5603511a-d427-4075-aaec-04f2a558c64d","head Pod":"fail-78fz8-head-7jx8g","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-78fz8-head-7jx8g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:45:15.032Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5603511a-d427-4075-aaec-04f2a558c64d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:45:15.032Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5603511a-d427-4075-aaec-04f2a558c64d","worker Pod":"fail-78fz8-small-group-worker-l8sb8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-78fz8-small-group-worker-l8sb8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:45:15.032Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5603511a-d427-4075-aaec-04f2a558c64d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:45:15.032Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5603511a-d427-4075-aaec-04f2a558c64d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:45:15.032Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5603511a-d427-4075-aaec-04f2a558c64d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:45:15.033Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5603511a-d427-4075-aaec-04f2a558c64d","podName":"fail-78fz8-small-group-worker-l8sb8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:45:15.033Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5603511a-d427-4075-aaec-04f2a558c64d","podName":"fail-78fz8-small-group-worker-l8sb8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:45:15.033Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5603511a-d427-4075-aaec-04f2a558c64d","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"fail-78fz8-head-7jx8g","serviceName":"fail-78fz8-head-svc"},"newHead":{"podIP":"10.132.0.23","serviceIP":"10.132.0.23","podName":"fail-78fz8-head-7jx8g","serviceName":"fail-78fz8-head-svc"}}
{"level":"info","ts":"2026-06-05T09:45:15.033Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5603511a-d427-4075-aaec-04f2a558c64d","name":"fail-78fz8","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:45:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-78fz8-head-7jx8g","serviceName":"fail-78fz8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:45:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.23","serviceIP":"10.132.0.23","podName":"fail-78fz8-head-7jx8g","serviceName":"fail-78fz8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:45:15.044Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"f57370c6-d5e4-43b9-9992-d10eb423e1fa","cluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:15.045Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"5a300c47-91d8-46b1-bc36-1e0ae04a052b","rayCluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:15.045Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"b33aab2f-7314-4dc5-94b2-2d4d52ab68f3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:45:15.045Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"b33aab2f-7314-4dc5-94b2-2d4d52ab68f3","name":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:15.045Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"b33aab2f-7314-4dc5-94b2-2d4d52ab68f3","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:15.045Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"5a300c47-91d8-46b1-bc36-1e0ae04a052b","pod":"fail-78fz8-head-7jx8g","ip":"10.132.0.23"}
{"level":"info","ts":"2026-06-05T09:45:15.045Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"5a300c47-91d8-46b1-bc36-1e0ae04a052b","pod":"fail-78fz8-small-group-worker-l8sb8","ip":"10.133.0.45"}
{"level":"info","ts":"2026-06-05T09:45:15.045Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"b33aab2f-7314-4dc5-94b2-2d4d52ab68f3"}
{"level":"info","ts":"2026-06-05T09:45:15.045Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"b33aab2f-7314-4dc5-94b2-2d4d52ab68f3","RayCluster":"fail-78fz8","State":""}
{"level":"info","ts":"2026-06-05T09:45:15.055Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"5a300c47-91d8-46b1-bc36-1e0ae04a052b","certificate":"ray-head-cert-fail-78fz8","dnsNames":["fail-78fz8-head-svc","localhost","fail-78fz8-head-svc.test-ns-thrrw.svc","fail-78fz8-head-svc.test-ns-thrrw.svc.cluster.local"],"ipAddresses":["10.132.0.23","10.133.0.45","127.0.0.1"]}
{"level":"info","ts":"2026-06-05T09:45:15.057Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"f57370c6-d5e4-43b9-9992-d10eb423e1fa","name":"fail-78fz8-head"}
{"level":"info","ts":"2026-06-05T09:45:15.068Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"5a300c47-91d8-46b1-bc36-1e0ae04a052b","certificate":"ray-worker-cert-fail-78fz8","dnsNames":["fail-78fz8-worker-svc","localhost","fail-78fz8-small-group","fail-78fz8-worker-svc.test-ns-thrrw.svc","fail-78fz8-worker-svc.test-ns-thrrw.svc.cluster.local","*.fail-78fz8-worker-svc.test-ns-thrrw.svc","*.fail-78fz8-worker-svc.test-ns-thrrw.svc.cluster.local","*-worker-*.test-ns-thrrw.svc","*-worker-*.test-ns-thrrw.svc.cluster.local"],"ipAddresses":["10.132.0.23","10.133.0.45","127.0.0.1"]}
{"level":"info","ts":"2026-06-05T09:45:15.068Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"5a300c47-91d8-46b1-bc36-1e0ae04a052b","rayCluster":"fail-78fz8","podIPs":["10.132.0.23","10.133.0.45"]}
{"level":"info","ts":"2026-06-05T09:45:15.071Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"f57370c6-d5e4-43b9-9992-d10eb423e1fa","name":"fail-78fz8-workers"}
{"level":"info","ts":"2026-06-05T09:45:15.071Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"f57370c6-d5e4-43b9-9992-d10eb423e1fa","cluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:17.045Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"24b0745c-5991-46a6-9aa8-fbcb01d2f151"}
{"level":"info","ts":"2026-06-05T09:45:17.045Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"24b0745c-5991-46a6-9aa8-fbcb01d2f151"}
{"level":"info","ts":"2026-06-05T09:45:17.045Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"24b0745c-5991-46a6-9aa8-fbcb01d2f151","1 head service found":"fail-78fz8-head-svc"}
{"level":"info","ts":"2026-06-05T09:45:17.045Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"24b0745c-5991-46a6-9aa8-fbcb01d2f151","Found 1 head Pod":"fail-78fz8-head-7jx8g","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:45:17.045Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"24b0745c-5991-46a6-9aa8-fbcb01d2f151","head Pod":"fail-78fz8-head-7jx8g","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-78fz8-head-7jx8g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:45:17.045Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"24b0745c-5991-46a6-9aa8-fbcb01d2f151","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:45:17.045Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"24b0745c-5991-46a6-9aa8-fbcb01d2f151","worker Pod":"fail-78fz8-small-group-worker-l8sb8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-78fz8-small-group-worker-l8sb8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:45:17.045Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"24b0745c-5991-46a6-9aa8-fbcb01d2f151","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:45:17.045Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"24b0745c-5991-46a6-9aa8-fbcb01d2f151","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:45:17.045Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"24b0745c-5991-46a6-9aa8-fbcb01d2f151","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:45:17.045Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"24b0745c-5991-46a6-9aa8-fbcb01d2f151","podName":"fail-78fz8-head-7jx8g","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:45:17.045Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"24b0745c-5991-46a6-9aa8-fbcb01d2f151","podName":"fail-78fz8-head-7jx8g","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:45:17.045Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"24b0745c-5991-46a6-9aa8-fbcb01d2f151","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:45:17.045Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"24b0745c-5991-46a6-9aa8-fbcb01d2f151","seconds":300}
{"level":"info","ts":"2026-06-05T09:45:17.306Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"1fcd4161-e496-412f-9fb2-002136c7fd12","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:45:17.306Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"1fcd4161-e496-412f-9fb2-002136c7fd12","name":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:17.306Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"1fcd4161-e496-412f-9fb2-002136c7fd12","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:17.306Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"1fcd4161-e496-412f-9fb2-002136c7fd12"}
{"level":"info","ts":"2026-06-05T09:45:17.306Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"1fcd4161-e496-412f-9fb2-002136c7fd12","RayCluster":"fail-78fz8","State":""}
{"level":"info","ts":"2026-06-05T09:45:20.055Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"1e3e4e90-f7bc-4606-a19a-c477f6e68a49"}
{"level":"info","ts":"2026-06-05T09:45:20.055Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"1e3e4e90-f7bc-4606-a19a-c477f6e68a49"}
{"level":"info","ts":"2026-06-05T09:45:20.055Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"1e3e4e90-f7bc-4606-a19a-c477f6e68a49","1 head service found":"fail-78fz8-head-svc"}
{"level":"info","ts":"2026-06-05T09:45:20.055Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"1e3e4e90-f7bc-4606-a19a-c477f6e68a49","Found 1 head Pod":"fail-78fz8-head-7jx8g","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:45:20.055Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"1e3e4e90-f7bc-4606-a19a-c477f6e68a49","head Pod":"fail-78fz8-head-7jx8g","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-78fz8-head-7jx8g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:45:20.055Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"1e3e4e90-f7bc-4606-a19a-c477f6e68a49","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:45:20.055Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"1e3e4e90-f7bc-4606-a19a-c477f6e68a49","worker Pod":"fail-78fz8-small-group-worker-l8sb8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-78fz8-small-group-worker-l8sb8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:45:20.055Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"1e3e4e90-f7bc-4606-a19a-c477f6e68a49","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:45:20.055Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"1e3e4e90-f7bc-4606-a19a-c477f6e68a49","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:45:20.055Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"1e3e4e90-f7bc-4606-a19a-c477f6e68a49","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:45:20.055Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"1e3e4e90-f7bc-4606-a19a-c477f6e68a49","podName":"fail-78fz8-head-7jx8g","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:45:20.056Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"1e3e4e90-f7bc-4606-a19a-c477f6e68a49","podName":"fail-78fz8-head-7jx8g","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:45:20.056Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"1e3e4e90-f7bc-4606-a19a-c477f6e68a49","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:45:20.056Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"1e3e4e90-f7bc-4606-a19a-c477f6e68a49","seconds":300}
{"level":"info","ts":"2026-06-05T09:45:20.307Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"2d00204a-d286-43a8-b8a8-4b3c6a6e406f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:45:20.307Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"2d00204a-d286-43a8-b8a8-4b3c6a6e406f","name":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:20.307Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"2d00204a-d286-43a8-b8a8-4b3c6a6e406f","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:20.307Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"2d00204a-d286-43a8-b8a8-4b3c6a6e406f"}
{"level":"info","ts":"2026-06-05T09:45:20.307Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"2d00204a-d286-43a8-b8a8-4b3c6a6e406f","RayCluster":"fail-78fz8","State":""}
{"level":"info","ts":"2026-06-05T09:45:21.066Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"8f5ecb7e-6be7-497b-82d5-39294b5ab097"}
{"level":"info","ts":"2026-06-05T09:45:21.066Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"8f5ecb7e-6be7-497b-82d5-39294b5ab097"}
{"level":"info","ts":"2026-06-05T09:45:21.066Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"8f5ecb7e-6be7-497b-82d5-39294b5ab097","1 head service found":"fail-78fz8-head-svc"}
{"level":"info","ts":"2026-06-05T09:45:21.066Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"8f5ecb7e-6be7-497b-82d5-39294b5ab097","Found 1 head Pod":"fail-78fz8-head-7jx8g","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:45:21.066Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"8f5ecb7e-6be7-497b-82d5-39294b5ab097","head Pod":"fail-78fz8-head-7jx8g","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-78fz8-head-7jx8g. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:45:21.066Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"8f5ecb7e-6be7-497b-82d5-39294b5ab097","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:45:21.066Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"8f5ecb7e-6be7-497b-82d5-39294b5ab097","worker Pod":"fail-78fz8-small-group-worker-l8sb8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-78fz8-small-group-worker-l8sb8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:45:21.066Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"8f5ecb7e-6be7-497b-82d5-39294b5ab097","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:45:21.066Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"8f5ecb7e-6be7-497b-82d5-39294b5ab097","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:45:21.066Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"8f5ecb7e-6be7-497b-82d5-39294b5ab097","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:45:21.067Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"8f5ecb7e-6be7-497b-82d5-39294b5ab097","podName":"fail-78fz8-head-7jx8g","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:45:14Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:45:21.067Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"8f5ecb7e-6be7-497b-82d5-39294b5ab097","podName":"fail-78fz8-head-7jx8g","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:45:14Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:45:21.067Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"8f5ecb7e-6be7-497b-82d5-39294b5ab097","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-05T09:45:21.067Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"8f5ecb7e-6be7-497b-82d5-39294b5ab097","name":"fail-78fz8","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:45:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.23","serviceIP":"10.132.0.23","podName":"fail-78fz8-head-7jx8g","serviceName":"fail-78fz8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:45:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.23","serviceIP":"10.132.0.23","podName":"fail-78fz8-head-7jx8g","serviceName":"fail-78fz8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:45:21.082Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"22fbbea6-c84b-45ff-9164-6cad5f8727fc","rayCluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:21.082Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"a557e1aa-dc45-4d83-a1e9-a4d8b35acd0d","cluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:21.082Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"22fbbea6-c84b-45ff-9164-6cad5f8727fc","pod":"fail-78fz8-head-7jx8g","ip":"10.132.0.23"}
{"level":"info","ts":"2026-06-05T09:45:21.082Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"22fbbea6-c84b-45ff-9164-6cad5f8727fc","pod":"fail-78fz8-small-group-worker-l8sb8","ip":"10.133.0.45"}
{"level":"info","ts":"2026-06-05T09:45:21.082Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"22fbbea6-c84b-45ff-9164-6cad5f8727fc","rayCluster":"fail-78fz8","podIPs":["10.132.0.23","10.133.0.45"]}
{"level":"info","ts":"2026-06-05T09:45:21.082Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"2b53bcf0-0d8a-463f-bf81-d5c21e9f01d3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:45:21.082Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"2b53bcf0-0d8a-463f-bf81-d5c21e9f01d3","name":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:21.082Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"2b53bcf0-0d8a-463f-bf81-d5c21e9f01d3","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:21.082Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"2b53bcf0-0d8a-463f-bf81-d5c21e9f01d3"}
{"level":"info","ts":"2026-06-05T09:45:21.082Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"2b53bcf0-0d8a-463f-bf81-d5c21e9f01d3","RayCluster":"fail-78fz8","State":""}
{"level":"info","ts":"2026-06-05T09:45:21.097Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"a557e1aa-dc45-4d83-a1e9-a4d8b35acd0d","name":"fail-78fz8-head"}
{"level":"info","ts":"2026-06-05T09:45:21.114Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"a557e1aa-dc45-4d83-a1e9-a4d8b35acd0d","name":"fail-78fz8-workers"}
{"level":"info","ts":"2026-06-05T09:45:21.114Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"a557e1aa-dc45-4d83-a1e9-a4d8b35acd0d","cluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:23.082Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5a7f5f6f-9dbc-4055-a95a-afdd683adbed"}
{"level":"info","ts":"2026-06-05T09:45:23.082Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5a7f5f6f-9dbc-4055-a95a-afdd683adbed"}
{"level":"info","ts":"2026-06-05T09:45:23.082Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5a7f5f6f-9dbc-4055-a95a-afdd683adbed","1 head service found":"fail-78fz8-head-svc"}
{"level":"info","ts":"2026-06-05T09:45:23.082Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5a7f5f6f-9dbc-4055-a95a-afdd683adbed","Found 1 head Pod":"fail-78fz8-head-7jx8g","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:45:23.082Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5a7f5f6f-9dbc-4055-a95a-afdd683adbed","head Pod":"fail-78fz8-head-7jx8g","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-78fz8-head-7jx8g. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:45:23.082Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5a7f5f6f-9dbc-4055-a95a-afdd683adbed","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:45:23.082Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5a7f5f6f-9dbc-4055-a95a-afdd683adbed","worker Pod":"fail-78fz8-small-group-worker-l8sb8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-78fz8-small-group-worker-l8sb8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:45:23.082Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5a7f5f6f-9dbc-4055-a95a-afdd683adbed","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:45:23.082Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5a7f5f6f-9dbc-4055-a95a-afdd683adbed","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:45:23.082Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5a7f5f6f-9dbc-4055-a95a-afdd683adbed","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:45:23.082Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5a7f5f6f-9dbc-4055-a95a-afdd683adbed","podName":"fail-78fz8-small-group-worker-l8sb8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:45:23.082Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5a7f5f6f-9dbc-4055-a95a-afdd683adbed","podName":"fail-78fz8-small-group-worker-l8sb8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:45:23.082Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5a7f5f6f-9dbc-4055-a95a-afdd683adbed","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:45:23.082Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"5a7f5f6f-9dbc-4055-a95a-afdd683adbed","seconds":300}
{"level":"info","ts":"2026-06-05T09:45:23.308Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"b96474d6-2396-474d-b877-8edbba8c43fa","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:45:23.308Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"b96474d6-2396-474d-b877-8edbba8c43fa","name":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:23.308Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"b96474d6-2396-474d-b877-8edbba8c43fa","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:23.308Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"b96474d6-2396-474d-b877-8edbba8c43fa"}
{"level":"info","ts":"2026-06-05T09:45:23.308Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"b96474d6-2396-474d-b877-8edbba8c43fa","RayCluster":"fail-78fz8","State":""}
{"level":"info","ts":"2026-06-05T09:45:24.971Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"1a245259-72de-45ef-af09-d7329bbf3aad"}
{"level":"info","ts":"2026-06-05T09:45:24.971Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"1a245259-72de-45ef-af09-d7329bbf3aad"}
{"level":"info","ts":"2026-06-05T09:45:24.971Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"1a245259-72de-45ef-af09-d7329bbf3aad","1 head service found":"fail-78fz8-head-svc"}
{"level":"info","ts":"2026-06-05T09:45:24.971Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"1a245259-72de-45ef-af09-d7329bbf3aad","Found 1 head Pod":"fail-78fz8-head-7jx8g","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:45:24.972Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"1a245259-72de-45ef-af09-d7329bbf3aad","head Pod":"fail-78fz8-head-7jx8g","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-78fz8-head-7jx8g. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:45:24.972Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"1a245259-72de-45ef-af09-d7329bbf3aad","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:45:24.972Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"1a245259-72de-45ef-af09-d7329bbf3aad","worker Pod":"fail-78fz8-small-group-worker-l8sb8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-78fz8-small-group-worker-l8sb8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:45:24.972Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"1a245259-72de-45ef-af09-d7329bbf3aad","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:45:24.972Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"1a245259-72de-45ef-af09-d7329bbf3aad","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:45:24.972Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"1a245259-72de-45ef-af09-d7329bbf3aad","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:45:24.972Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"1a245259-72de-45ef-af09-d7329bbf3aad","podName":"fail-78fz8-head-7jx8g","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:45:14Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:45:24.972Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"1a245259-72de-45ef-af09-d7329bbf3aad","podName":"fail-78fz8-head-7jx8g","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:45:14Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:45:24.972Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"1a245259-72de-45ef-af09-d7329bbf3aad","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:45:24.972Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"1a245259-72de-45ef-af09-d7329bbf3aad","seconds":300}
{"level":"info","ts":"2026-06-05T09:45:25.981Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"6794ddba-7fb7-4e55-984d-edd90fe20943"}
{"level":"info","ts":"2026-06-05T09:45:25.981Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"6794ddba-7fb7-4e55-984d-edd90fe20943"}
{"level":"info","ts":"2026-06-05T09:45:25.981Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"6794ddba-7fb7-4e55-984d-edd90fe20943","1 head service found":"fail-78fz8-head-svc"}
{"level":"info","ts":"2026-06-05T09:45:25.981Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"6794ddba-7fb7-4e55-984d-edd90fe20943","Found 1 head Pod":"fail-78fz8-head-7jx8g","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:45:25.981Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"6794ddba-7fb7-4e55-984d-edd90fe20943","head Pod":"fail-78fz8-head-7jx8g","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-78fz8-head-7jx8g. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:45:25.981Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"6794ddba-7fb7-4e55-984d-edd90fe20943","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:45:25.981Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"6794ddba-7fb7-4e55-984d-edd90fe20943","worker Pod":"fail-78fz8-small-group-worker-l8sb8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-78fz8-small-group-worker-l8sb8. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:45:25.981Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"6794ddba-7fb7-4e55-984d-edd90fe20943","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:45:25.981Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"6794ddba-7fb7-4e55-984d-edd90fe20943","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:45:25.981Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"6794ddba-7fb7-4e55-984d-edd90fe20943","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:45:25.981Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"6794ddba-7fb7-4e55-984d-edd90fe20943","podName":"fail-78fz8-head-7jx8g","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:45:14Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:45:25.981Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"6794ddba-7fb7-4e55-984d-edd90fe20943","podName":"fail-78fz8-head-7jx8g","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:45:14Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:45:25.981Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"6794ddba-7fb7-4e55-984d-edd90fe20943","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-05T09:45:25.981Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"6794ddba-7fb7-4e55-984d-edd90fe20943","name":"fail-78fz8","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:45:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.23","serviceIP":"10.132.0.23","podName":"fail-78fz8-head-7jx8g","serviceName":"fail-78fz8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:45:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.23","serviceIP":"10.132.0.23","podName":"fail-78fz8-head-7jx8g","serviceName":"fail-78fz8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:45:25.994Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"b31aecb6-8662-4f2c-9da4-03e03a63b10d","rayCluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:25.994Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"fe81f328-5c02-4bb1-a0f5-2ecca61281ab","cluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:25.994Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"b31aecb6-8662-4f2c-9da4-03e03a63b10d","pod":"fail-78fz8-head-7jx8g","ip":"10.132.0.23"}
{"level":"info","ts":"2026-06-05T09:45:25.994Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"b31aecb6-8662-4f2c-9da4-03e03a63b10d","pod":"fail-78fz8-small-group-worker-l8sb8","ip":"10.133.0.45"}
{"level":"info","ts":"2026-06-05T09:45:25.994Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"a9801624-7ed3-4f60-b530-34b1bc2bc950","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:45:25.994Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"a9801624-7ed3-4f60-b530-34b1bc2bc950","name":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:25.994Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"a9801624-7ed3-4f60-b530-34b1bc2bc950","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:25.994Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"b31aecb6-8662-4f2c-9da4-03e03a63b10d","rayCluster":"fail-78fz8","podIPs":["10.132.0.23","10.133.0.45"]}
{"level":"info","ts":"2026-06-05T09:45:25.994Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"a9801624-7ed3-4f60-b530-34b1bc2bc950"}
{"level":"info","ts":"2026-06-05T09:45:25.994Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"a9801624-7ed3-4f60-b530-34b1bc2bc950","RayCluster":"fail-78fz8","State":""}
{"level":"info","ts":"2026-06-05T09:45:26.007Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"fe81f328-5c02-4bb1-a0f5-2ecca61281ab","name":"fail-78fz8-head"}
{"level":"info","ts":"2026-06-05T09:45:26.020Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"fe81f328-5c02-4bb1-a0f5-2ecca61281ab","name":"fail-78fz8-workers"}
{"level":"info","ts":"2026-06-05T09:45:26.020Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"fe81f328-5c02-4bb1-a0f5-2ecca61281ab","cluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:26.309Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"fb82e7e1-37b3-4c38-832d-36e5a38376e8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:45:26.310Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"fb82e7e1-37b3-4c38-832d-36e5a38376e8","name":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:26.310Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"fb82e7e1-37b3-4c38-832d-36e5a38376e8","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:26.310Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"fb82e7e1-37b3-4c38-832d-36e5a38376e8"}
{"level":"info","ts":"2026-06-05T09:45:26.310Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"fb82e7e1-37b3-4c38-832d-36e5a38376e8","RayCluster":"fail-78fz8","State":""}
{"level":"info","ts":"2026-06-05T09:45:27.994Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"2268dea7-6832-4a48-aab2-9b907141f92e"}
{"level":"info","ts":"2026-06-05T09:45:27.994Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"2268dea7-6832-4a48-aab2-9b907141f92e"}
{"level":"info","ts":"2026-06-05T09:45:27.994Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"2268dea7-6832-4a48-aab2-9b907141f92e","1 head service found":"fail-78fz8-head-svc"}
{"level":"info","ts":"2026-06-05T09:45:27.994Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"2268dea7-6832-4a48-aab2-9b907141f92e","Found 1 head Pod":"fail-78fz8-head-7jx8g","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:45:27.994Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"2268dea7-6832-4a48-aab2-9b907141f92e","head Pod":"fail-78fz8-head-7jx8g","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-78fz8-head-7jx8g. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:45:27.994Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"2268dea7-6832-4a48-aab2-9b907141f92e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:45:27.994Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"2268dea7-6832-4a48-aab2-9b907141f92e","worker Pod":"fail-78fz8-small-group-worker-l8sb8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-78fz8-small-group-worker-l8sb8. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:45:27.994Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"2268dea7-6832-4a48-aab2-9b907141f92e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:45:27.994Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"2268dea7-6832-4a48-aab2-9b907141f92e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:45:27.994Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"2268dea7-6832-4a48-aab2-9b907141f92e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:45:27.995Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"2268dea7-6832-4a48-aab2-9b907141f92e","podName":"fail-78fz8-head-7jx8g","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:45:14Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:45:27.995Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"2268dea7-6832-4a48-aab2-9b907141f92e","podName":"fail-78fz8-head-7jx8g","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:45:14Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:45:27.995Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"2268dea7-6832-4a48-aab2-9b907141f92e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:45:27.995Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"2268dea7-6832-4a48-aab2-9b907141f92e","seconds":300}
{"level":"info","ts":"2026-06-05T09:45:29.311Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"001bde89-5fcb-4c9a-8fcf-70c0182aaf27","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:45:29.311Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"001bde89-5fcb-4c9a-8fcf-70c0182aaf27","name":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:29.311Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"001bde89-5fcb-4c9a-8fcf-70c0182aaf27","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:29.311Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"001bde89-5fcb-4c9a-8fcf-70c0182aaf27"}
{"level":"info","ts":"2026-06-05T09:45:29.311Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"001bde89-5fcb-4c9a-8fcf-70c0182aaf27","RayCluster":"fail-78fz8","State":""}
{"level":"info","ts":"2026-06-05T09:45:32.091Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"e4f28cdf-fd05-4276-a838-9ce11b27665b"}
{"level":"info","ts":"2026-06-05T09:45:32.091Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"e4f28cdf-fd05-4276-a838-9ce11b27665b"}
{"level":"info","ts":"2026-06-05T09:45:32.091Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"e4f28cdf-fd05-4276-a838-9ce11b27665b","1 head service found":"fail-78fz8-head-svc"}
{"level":"info","ts":"2026-06-05T09:45:32.091Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"e4f28cdf-fd05-4276-a838-9ce11b27665b","Found 1 head Pod":"fail-78fz8-head-7jx8g","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:45:32.091Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"e4f28cdf-fd05-4276-a838-9ce11b27665b","head Pod":"fail-78fz8-head-7jx8g","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-78fz8-head-7jx8g. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:45:32.091Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"e4f28cdf-fd05-4276-a838-9ce11b27665b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:45:32.091Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"e4f28cdf-fd05-4276-a838-9ce11b27665b","worker Pod":"fail-78fz8-small-group-worker-l8sb8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-78fz8-small-group-worker-l8sb8. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:45:32.091Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"e4f28cdf-fd05-4276-a838-9ce11b27665b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:45:32.091Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"e4f28cdf-fd05-4276-a838-9ce11b27665b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:45:32.091Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"e4f28cdf-fd05-4276-a838-9ce11b27665b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:45:32.091Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"e4f28cdf-fd05-4276-a838-9ce11b27665b","podName":"fail-78fz8-small-group-worker-l8sb8","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:45:14Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-05T09:45:32.091Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"e4f28cdf-fd05-4276-a838-9ce11b27665b","podName":"fail-78fz8-small-group-worker-l8sb8","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:45:14Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-05T09:45:32.091Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"e4f28cdf-fd05-4276-a838-9ce11b27665b","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:45:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-05T09:45:32.091Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"e4f28cdf-fd05-4276-a838-9ce11b27665b","name":"fail-78fz8","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:45:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.23","serviceIP":"10.132.0.23","podName":"fail-78fz8-head-7jx8g","serviceName":"fail-78fz8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:45:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.23","serviceIP":"10.132.0.23","podName":"fail-78fz8-head-7jx8g","serviceName":"fail-78fz8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:45:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:45:32.104Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"c93c6f93-9ed0-4806-b0f9-4a7227d68d26","rayCluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:32.105Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"1b776227-043c-4dbd-a2f0-9bbd9ba280ad","cluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:32.105Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"c93c6f93-9ed0-4806-b0f9-4a7227d68d26","pod":"fail-78fz8-head-7jx8g","ip":"10.132.0.23"}
{"level":"info","ts":"2026-06-05T09:45:32.105Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"c93c6f93-9ed0-4806-b0f9-4a7227d68d26","pod":"fail-78fz8-small-group-worker-l8sb8","ip":"10.133.0.45"}
{"level":"info","ts":"2026-06-05T09:45:32.105Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"197ed3bf-e9dc-466b-899d-14cb6c35938e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:45:32.105Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"197ed3bf-e9dc-466b-899d-14cb6c35938e","name":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:32.105Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"c93c6f93-9ed0-4806-b0f9-4a7227d68d26","rayCluster":"fail-78fz8","podIPs":["10.132.0.23","10.133.0.45"]}
{"level":"info","ts":"2026-06-05T09:45:32.105Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"197ed3bf-e9dc-466b-899d-14cb6c35938e","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:32.105Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"197ed3bf-e9dc-466b-899d-14cb6c35938e"}
{"level":"info","ts":"2026-06-05T09:45:32.105Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"197ed3bf-e9dc-466b-899d-14cb6c35938e","RayCluster":"fail-78fz8","State":""}
{"level":"info","ts":"2026-06-05T09:45:32.117Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"1b776227-043c-4dbd-a2f0-9bbd9ba280ad","name":"fail-78fz8-head"}
{"level":"info","ts":"2026-06-05T09:45:32.128Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"1b776227-043c-4dbd-a2f0-9bbd9ba280ad","name":"fail-78fz8-workers"}
{"level":"info","ts":"2026-06-05T09:45:32.128Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"1b776227-043c-4dbd-a2f0-9bbd9ba280ad","cluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:32.312Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"9f5e15ba-36a9-4488-a8b0-2e15fbab4d93","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:45:32.312Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"9f5e15ba-36a9-4488-a8b0-2e15fbab4d93","name":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:32.312Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"9f5e15ba-36a9-4488-a8b0-2e15fbab4d93","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:32.312Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"9f5e15ba-36a9-4488-a8b0-2e15fbab4d93"}
{"level":"info","ts":"2026-06-05T09:45:32.312Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"9f5e15ba-36a9-4488-a8b0-2e15fbab4d93","RayCluster":"fail-78fz8","State":""}
{"level":"info","ts":"2026-06-05T09:45:34.105Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"35cbc088-629c-40c7-a7cf-cce3207a4fad"}
{"level":"info","ts":"2026-06-05T09:45:34.105Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"35cbc088-629c-40c7-a7cf-cce3207a4fad"}
{"level":"info","ts":"2026-06-05T09:45:34.105Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"35cbc088-629c-40c7-a7cf-cce3207a4fad","1 head service found":"fail-78fz8-head-svc"}
{"level":"info","ts":"2026-06-05T09:45:34.105Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"35cbc088-629c-40c7-a7cf-cce3207a4fad","Found 1 head Pod":"fail-78fz8-head-7jx8g","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:45:34.105Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"35cbc088-629c-40c7-a7cf-cce3207a4fad","head Pod":"fail-78fz8-head-7jx8g","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-78fz8-head-7jx8g. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:45:34.105Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"35cbc088-629c-40c7-a7cf-cce3207a4fad","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:45:34.105Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"35cbc088-629c-40c7-a7cf-cce3207a4fad","worker Pod":"fail-78fz8-small-group-worker-l8sb8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-78fz8-small-group-worker-l8sb8. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:45:34.105Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"35cbc088-629c-40c7-a7cf-cce3207a4fad","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:45:34.105Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"35cbc088-629c-40c7-a7cf-cce3207a4fad","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:45:34.105Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"35cbc088-629c-40c7-a7cf-cce3207a4fad","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:45:34.105Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"35cbc088-629c-40c7-a7cf-cce3207a4fad","podName":"fail-78fz8-small-group-worker-l8sb8","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:45:14Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-05T09:45:34.105Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"35cbc088-629c-40c7-a7cf-cce3207a4fad","podName":"fail-78fz8-small-group-worker-l8sb8","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:45:14Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-05T09:45:34.105Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"35cbc088-629c-40c7-a7cf-cce3207a4fad","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:45:34.105Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"35cbc088-629c-40c7-a7cf-cce3207a4fad","seconds":300}
{"level":"info","ts":"2026-06-05T09:45:35.314Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"4980589b-3986-4619-9a7c-15bcbe8e5dfc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:45:35.314Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"4980589b-3986-4619-9a7c-15bcbe8e5dfc","name":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:35.314Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"4980589b-3986-4619-9a7c-15bcbe8e5dfc","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:35.314Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"4980589b-3986-4619-9a7c-15bcbe8e5dfc"}
{"level":"info","ts":"2026-06-05T09:45:35.314Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"4980589b-3986-4619-9a7c-15bcbe8e5dfc","RayCluster":"fail-78fz8","State":""}
{"level":"info","ts":"2026-06-05T09:45:36.553Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"bd1934fb-d698-42bf-8ebc-65c62f2d93d7","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:45:36.553Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"bd1934fb-d698-42bf-8ebc-65c62f2d93d7","pod":"raycluster-head-vjd7d","ip":"10.134.0.20"}
{"level":"info","ts":"2026-06-05T09:45:36.553Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"bd1934fb-d698-42bf-8ebc-65c62f2d93d7","pod":"raycluster-small-group-worker-j9pbw","ip":"10.133.0.35"}
{"level":"info","ts":"2026-06-05T09:45:36.553Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"bd1934fb-d698-42bf-8ebc-65c62f2d93d7","rayCluster":"raycluster","podIPs":["10.134.0.20","10.133.0.35"]}
{"level":"info","ts":"2026-06-05T09:45:37.005Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"3c20ed3c-210f-4735-994c-eaea85e4bf43"}
{"level":"info","ts":"2026-06-05T09:45:37.006Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"3c20ed3c-210f-4735-994c-eaea85e4bf43"}
{"level":"info","ts":"2026-06-05T09:45:37.006Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"3c20ed3c-210f-4735-994c-eaea85e4bf43","1 head service found":"fail-78fz8-head-svc"}
{"level":"info","ts":"2026-06-05T09:45:37.006Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"3c20ed3c-210f-4735-994c-eaea85e4bf43","Found 1 head Pod":"fail-78fz8-head-7jx8g","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:45:37.006Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"3c20ed3c-210f-4735-994c-eaea85e4bf43","head Pod":"fail-78fz8-head-7jx8g","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-78fz8-head-7jx8g. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:45:37.006Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"3c20ed3c-210f-4735-994c-eaea85e4bf43","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:45:37.006Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"3c20ed3c-210f-4735-994c-eaea85e4bf43","worker Pod":"fail-78fz8-small-group-worker-l8sb8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-78fz8-small-group-worker-l8sb8. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:45:37.006Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"3c20ed3c-210f-4735-994c-eaea85e4bf43","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:45:37.006Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"3c20ed3c-210f-4735-994c-eaea85e4bf43","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:45:37.006Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"3c20ed3c-210f-4735-994c-eaea85e4bf43","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:45:37.006Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"3c20ed3c-210f-4735-994c-eaea85e4bf43","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-06-05T09:45:37.006Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"3c20ed3c-210f-4735-994c-eaea85e4bf43","name":"fail-78fz8","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:45:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.23","serviceIP":"10.132.0.23","podName":"fail-78fz8-head-7jx8g","serviceName":"fail-78fz8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:45:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:45:37Z","stateTransitionTimes":{"ready":"2026-06-05T09:45:37Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.23","serviceIP":"10.132.0.23","podName":"fail-78fz8-head-7jx8g","serviceName":"fail-78fz8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:45:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:45:37Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:45:37.018Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"f82759a6-5fe1-4158-933b-b667e7d79c9b","rayCluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:37.018Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"f5ca38a2-6513-477a-912b-22e227e58ddd","cluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:37.019Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"43c16b35-9bd2-4ce0-b536-df84a9325e7f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:45:37.019Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"43c16b35-9bd2-4ce0-b536-df84a9325e7f","name":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:37.019Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"43c16b35-9bd2-4ce0-b536-df84a9325e7f","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:37.019Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"f82759a6-5fe1-4158-933b-b667e7d79c9b","pod":"fail-78fz8-head-7jx8g","ip":"10.132.0.23"}
{"level":"info","ts":"2026-06-05T09:45:37.019Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"f82759a6-5fe1-4158-933b-b667e7d79c9b","pod":"fail-78fz8-small-group-worker-l8sb8","ip":"10.133.0.45"}
{"level":"info","ts":"2026-06-05T09:45:37.019Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"f82759a6-5fe1-4158-933b-b667e7d79c9b","rayCluster":"fail-78fz8","podIPs":["10.132.0.23","10.133.0.45"]}
{"level":"info","ts":"2026-06-05T09:45:37.019Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"43c16b35-9bd2-4ce0-b536-df84a9325e7f"}
{"level":"info","ts":"2026-06-05T09:45:37.019Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"43c16b35-9bd2-4ce0-b536-df84a9325e7f","head service name":"fail-78fz8-head-svc","namespace":"test-ns-thrrw"}
{"level":"info","ts":"2026-06-05T09:45:37.019Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"43c16b35-9bd2-4ce0-b536-df84a9325e7f","head service URL":"fail-78fz8-head-svc.test-ns-thrrw.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-05T09:45:37.019Z","logger":"controllers.RayJob","msg":"user-provided submitter template is used; the first container is assumed to be the submitter","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"43c16b35-9bd2-4ce0-b536-df84a9325e7f"}
{"level":"info","ts":"2026-06-05T09:45:37.019Z","logger":"controllers.RayJob","msg":"No command is specified in the user-provided template. Default command is used","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"43c16b35-9bd2-4ce0-b536-df84a9325e7f","command":["if","!","ray","job","status","--address","http://fail-78fz8-head-svc.test-ns-thrrw.svc.cluster.local:8265","fail-qjbjr",">/dev/null","2>&1",";","then","ray","job","submit","--address","http://fail-78fz8-head-svc.test-ns-thrrw.svc.cluster.local:8265","--no-wait","--submission-id","fail-qjbjr","--","python /home/ray/jobs/fail.py",";","fi",";","ray","job","logs","--address","http://fail-78fz8-head-svc.test-ns-thrrw.svc.cluster.local:8265","--follow","fail-qjbjr"]}
{"level":"info","ts":"2026-06-05T09:45:37.027Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"43c16b35-9bd2-4ce0-b536-df84a9325e7f","Kubernetes Job":"fail"}
{"level":"info","ts":"2026-06-05T09:45:37.027Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"43c16b35-9bd2-4ce0-b536-df84a9325e7f","SubmissionMode":"K8sJobMode","RayCluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:37.028Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"43c16b35-9bd2-4ce0-b536-df84a9325e7f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-qjbjr","rayClusterName":"fail-78fz8","jobDeploymentStatus":"Initializing","startTime":"2026-06-05T09:45:11Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-qjbjr","rayClusterName":"fail-78fz8","dashboardURL":"fail-78fz8-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-05T09:45:11Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:45:37.028Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"43c16b35-9bd2-4ce0-b536-df84a9325e7f","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-05T09:45:37.034Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"f5ca38a2-6513-477a-912b-22e227e58ddd","name":"fail-78fz8-head"}
{"level":"info","ts":"2026-06-05T09:45:37.038Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"c0aea3fa-2bd5-40de-b360-bbaf207a316c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:45:37.038Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"c0aea3fa-2bd5-40de-b360-bbaf207a316c","name":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:37.038Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"c0aea3fa-2bd5-40de-b360-bbaf207a316c","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:37.039Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"c0aea3fa-2bd5-40de-b360-bbaf207a316c"}
{"level":"info","ts":"2026-06-05T09:45:37.039Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"c0aea3fa-2bd5-40de-b360-bbaf207a316c","head service name":"fail-78fz8-head-svc","namespace":"test-ns-thrrw"}
{"level":"info","ts":"2026-06-05T09:45:37.039Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"c0aea3fa-2bd5-40de-b360-bbaf207a316c","head service URL":"fail-78fz8-head-svc.test-ns-thrrw.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-05T09:45:37.039Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job for RayJob already exists","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"c0aea3fa-2bd5-40de-b360-bbaf207a316c","Kubernetes Job":"fail"}
{"level":"info","ts":"2026-06-05T09:45:37.039Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"c0aea3fa-2bd5-40de-b360-bbaf207a316c","SubmissionMode":"K8sJobMode","RayCluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:37.039Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"c0aea3fa-2bd5-40de-b360-bbaf207a316c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-qjbjr","rayClusterName":"fail-78fz8","jobDeploymentStatus":"Initializing","startTime":"2026-06-05T09:45:11Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-qjbjr","rayClusterName":"fail-78fz8","dashboardURL":"fail-78fz8-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-05T09:45:11Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:45:37.039Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"c0aea3fa-2bd5-40de-b360-bbaf207a316c","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-05T09:45:37.049Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"c0aea3fa-2bd5-40de-b360-bbaf207a316c","error":"Operation cannot be fulfilled on rayjobs.ray.io \"fail\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-06-05T09:45:37.049Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"c0aea3fa-2bd5-40de-b360-bbaf207a316c"}
{"level":"error","ts":"2026-06-05T09:45:37.049Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"c0aea3fa-2bd5-40de-b360-bbaf207a316c","error":"Operation cannot be fulfilled on rayjobs.ray.io \"fail\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:37.049Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"01e0bcc9-b10a-4cbd-b907-84ad7edde053","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:45:37.049Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"01e0bcc9-b10a-4cbd-b907-84ad7edde053","name":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:37.049Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"01e0bcc9-b10a-4cbd-b907-84ad7edde053","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:37.049Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"01e0bcc9-b10a-4cbd-b907-84ad7edde053"}
{"level":"info","ts":"2026-06-05T09:45:37.056Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"f5ca38a2-6513-477a-912b-22e227e58ddd","name":"fail-78fz8-workers"}
{"level":"info","ts":"2026-06-05T09:45:37.056Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"f5ca38a2-6513-477a-912b-22e227e58ddd","cluster":"fail-78fz8"}
{"level":"error","ts":"2026-06-05T09:45:37.105Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"01e0bcc9-b10a-4cbd-b907-84ad7edde053","JobId":"fail-qjbjr","error":"Job fail-qjbjr does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:37.106Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"01e0bcc9-b10a-4cbd-b907-84ad7edde053"}
{"level":"error","ts":"2026-06-05T09:45:37.106Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"01e0bcc9-b10a-4cbd-b907-84ad7edde053","error":"Job fail-qjbjr does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:37.106Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"8db670ad-f53a-451b-a6d0-fa455e8d6fae","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:45:37.106Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"8db670ad-f53a-451b-a6d0-fa455e8d6fae","name":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:37.106Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"8db670ad-f53a-451b-a6d0-fa455e8d6fae","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:37.106Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"8db670ad-f53a-451b-a6d0-fa455e8d6fae"}
{"level":"error","ts":"2026-06-05T09:45:37.111Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"8db670ad-f53a-451b-a6d0-fa455e8d6fae","JobId":"fail-qjbjr","error":"Job fail-qjbjr does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:37.111Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"8db670ad-f53a-451b-a6d0-fa455e8d6fae"}
{"level":"error","ts":"2026-06-05T09:45:37.111Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"8db670ad-f53a-451b-a6d0-fa455e8d6fae","error":"Job fail-qjbjr does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:37.117Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"5589341a-535b-4848-9a9d-b1f5c89ad344","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:45:37.117Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"5589341a-535b-4848-9a9d-b1f5c89ad344","name":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:37.117Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"5589341a-535b-4848-9a9d-b1f5c89ad344","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:37.117Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"5589341a-535b-4848-9a9d-b1f5c89ad344"}
{"level":"error","ts":"2026-06-05T09:45:37.121Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"5589341a-535b-4848-9a9d-b1f5c89ad344","JobId":"fail-qjbjr","error":"Job fail-qjbjr does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:37.122Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"5589341a-535b-4848-9a9d-b1f5c89ad344"}
{"level":"error","ts":"2026-06-05T09:45:37.122Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"5589341a-535b-4848-9a9d-b1f5c89ad344","error":"Job fail-qjbjr does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:37.162Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"85e7774f-a4ff-45c4-9f60-d1dbf8c989a6","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:45:37.162Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"85e7774f-a4ff-45c4-9f60-d1dbf8c989a6","name":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:37.162Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"85e7774f-a4ff-45c4-9f60-d1dbf8c989a6","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:37.162Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"85e7774f-a4ff-45c4-9f60-d1dbf8c989a6"}
{"level":"error","ts":"2026-06-05T09:45:37.167Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"85e7774f-a4ff-45c4-9f60-d1dbf8c989a6","JobId":"fail-qjbjr","error":"Job fail-qjbjr does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:37.167Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"85e7774f-a4ff-45c4-9f60-d1dbf8c989a6"}
{"level":"error","ts":"2026-06-05T09:45:37.167Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"85e7774f-a4ff-45c4-9f60-d1dbf8c989a6","error":"Job fail-qjbjr does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:37.247Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"2c9571d0-2c60-41e1-a7b0-5d6a011c83e7","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:45:37.247Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"2c9571d0-2c60-41e1-a7b0-5d6a011c83e7","name":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:37.247Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"2c9571d0-2c60-41e1-a7b0-5d6a011c83e7","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:37.248Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"2c9571d0-2c60-41e1-a7b0-5d6a011c83e7"}
{"level":"error","ts":"2026-06-05T09:45:37.253Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"2c9571d0-2c60-41e1-a7b0-5d6a011c83e7","JobId":"fail-qjbjr","error":"Job fail-qjbjr does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:37.253Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"2c9571d0-2c60-41e1-a7b0-5d6a011c83e7"}
{"level":"error","ts":"2026-06-05T09:45:37.253Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"2c9571d0-2c60-41e1-a7b0-5d6a011c83e7","error":"Job fail-qjbjr does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:37.413Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"01f14061-8965-46fb-bdad-a979932ddd23","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:45:37.413Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"01f14061-8965-46fb-bdad-a979932ddd23","name":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:37.413Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"01f14061-8965-46fb-bdad-a979932ddd23","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:37.413Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"01f14061-8965-46fb-bdad-a979932ddd23"}
{"level":"error","ts":"2026-06-05T09:45:37.419Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"01f14061-8965-46fb-bdad-a979932ddd23","JobId":"fail-qjbjr","error":"Job fail-qjbjr does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:37.419Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"01f14061-8965-46fb-bdad-a979932ddd23"}
{"level":"error","ts":"2026-06-05T09:45:37.419Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"01f14061-8965-46fb-bdad-a979932ddd23","error":"Job fail-qjbjr does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:37.739Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"e24b6973-1c94-4856-9ef3-c8dcfbbeed3d","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:45:37.739Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"e24b6973-1c94-4856-9ef3-c8dcfbbeed3d","name":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:37.739Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"e24b6973-1c94-4856-9ef3-c8dcfbbeed3d","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:37.740Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"e24b6973-1c94-4856-9ef3-c8dcfbbeed3d"}
{"level":"error","ts":"2026-06-05T09:45:37.745Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"e24b6973-1c94-4856-9ef3-c8dcfbbeed3d","JobId":"fail-qjbjr","error":"Job fail-qjbjr does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:37.745Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"e24b6973-1c94-4856-9ef3-c8dcfbbeed3d"}
{"level":"error","ts":"2026-06-05T09:45:37.745Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"e24b6973-1c94-4856-9ef3-c8dcfbbeed3d","error":"Job fail-qjbjr does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:38.051Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"25af816f-701e-4d57-95a0-7df04e18da72","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:45:38.051Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"25af816f-701e-4d57-95a0-7df04e18da72","name":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:38.051Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"25af816f-701e-4d57-95a0-7df04e18da72","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:38.051Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"25af816f-701e-4d57-95a0-7df04e18da72"}
{"level":"error","ts":"2026-06-05T09:45:38.056Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"25af816f-701e-4d57-95a0-7df04e18da72","JobId":"fail-qjbjr","error":"Job fail-qjbjr does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:38.056Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"25af816f-701e-4d57-95a0-7df04e18da72"}
{"level":"error","ts":"2026-06-05T09:45:38.056Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"25af816f-701e-4d57-95a0-7df04e18da72","error":"Job fail-qjbjr does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:38.386Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"46a333d4-cdab-46fe-9e2c-c9993716584c","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:45:38.386Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"46a333d4-cdab-46fe-9e2c-c9993716584c","name":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:38.386Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"46a333d4-cdab-46fe-9e2c-c9993716584c","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:38.386Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"46a333d4-cdab-46fe-9e2c-c9993716584c"}
{"level":"error","ts":"2026-06-05T09:45:38.392Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"46a333d4-cdab-46fe-9e2c-c9993716584c","JobId":"fail-qjbjr","error":"Job fail-qjbjr does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:38.392Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"46a333d4-cdab-46fe-9e2c-c9993716584c"}
{"level":"error","ts":"2026-06-05T09:45:38.392Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"46a333d4-cdab-46fe-9e2c-c9993716584c","error":"Job fail-qjbjr does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:39.019Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"4fc760ef-93e2-4eb3-a664-94fad7545204"}
{"level":"info","ts":"2026-06-05T09:45:39.019Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"4fc760ef-93e2-4eb3-a664-94fad7545204"}
{"level":"info","ts":"2026-06-05T09:45:39.019Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"4fc760ef-93e2-4eb3-a664-94fad7545204","1 head service found":"fail-78fz8-head-svc"}
{"level":"info","ts":"2026-06-05T09:45:39.019Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"4fc760ef-93e2-4eb3-a664-94fad7545204","Found 1 head Pod":"fail-78fz8-head-7jx8g","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:45:39.019Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"4fc760ef-93e2-4eb3-a664-94fad7545204","head Pod":"fail-78fz8-head-7jx8g","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-78fz8-head-7jx8g. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:45:39.019Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"4fc760ef-93e2-4eb3-a664-94fad7545204","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:45:39.019Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"4fc760ef-93e2-4eb3-a664-94fad7545204","worker Pod":"fail-78fz8-small-group-worker-l8sb8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-78fz8-small-group-worker-l8sb8. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:45:39.019Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"4fc760ef-93e2-4eb3-a664-94fad7545204","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:45:39.019Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"4fc760ef-93e2-4eb3-a664-94fad7545204","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:45:39.019Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"4fc760ef-93e2-4eb3-a664-94fad7545204","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:45:39.020Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"4fc760ef-93e2-4eb3-a664-94fad7545204","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:45:39.020Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"4fc760ef-93e2-4eb3-a664-94fad7545204","seconds":300}
{"level":"info","ts":"2026-06-05T09:45:40.953Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"cbbb075c-e081-4e6b-a58d-0b424c5932bf","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:45:40.953Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"cbbb075c-e081-4e6b-a58d-0b424c5932bf","name":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:40.953Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"cbbb075c-e081-4e6b-a58d-0b424c5932bf","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:40.953Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"cbbb075c-e081-4e6b-a58d-0b424c5932bf"}
{"level":"error","ts":"2026-06-05T09:45:40.958Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"cbbb075c-e081-4e6b-a58d-0b424c5932bf","JobId":"fail-qjbjr","error":"Job fail-qjbjr does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:40.958Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"cbbb075c-e081-4e6b-a58d-0b424c5932bf"}
{"level":"error","ts":"2026-06-05T09:45:40.958Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"cbbb075c-e081-4e6b-a58d-0b424c5932bf","error":"Job fail-qjbjr does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:41.418Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"38128fb5-104c-4f58-9c38-4195158055d0","rayCluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:41.419Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"38128fb5-104c-4f58-9c38-4195158055d0","pod":"fail-78fz8-head-7jx8g","ip":"10.132.0.23"}
{"level":"info","ts":"2026-06-05T09:45:41.419Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"38128fb5-104c-4f58-9c38-4195158055d0","pod":"fail-78fz8-small-group-worker-l8sb8","ip":"10.133.0.45"}
{"level":"info","ts":"2026-06-05T09:45:41.419Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"38128fb5-104c-4f58-9c38-4195158055d0","rayCluster":"fail-78fz8","podIPs":["10.132.0.23","10.133.0.45"]}
{"level":"info","ts":"2026-06-05T09:45:46.079Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"20e3dbcb-a89b-4c31-9a67-6a8aeab04152","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:45:46.079Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"20e3dbcb-a89b-4c31-9a67-6a8aeab04152","name":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:46.079Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"20e3dbcb-a89b-4c31-9a67-6a8aeab04152","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:46.080Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"20e3dbcb-a89b-4c31-9a67-6a8aeab04152"}
{"level":"info","ts":"2026-06-05T09:45:46.086Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"20e3dbcb-a89b-4c31-9a67-6a8aeab04152","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-qjbjr","rayClusterName":"fail-78fz8","dashboardURL":"fail-78fz8-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-05T09:45:11Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:45:44Z"},"jobId":"fail-qjbjr","rayClusterName":"fail-78fz8","dashboardURL":"fail-78fz8-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-05T09:45:11Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:45:37Z","stateTransitionTimes":{"ready":"2026-06-05T09:45:37Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.23","serviceIP":"10.132.0.23","podName":"fail-78fz8-head-7jx8g","serviceName":"fail-78fz8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:45:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:45:37Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:45:46.086Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"20e3dbcb-a89b-4c31-9a67-6a8aeab04152","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-05T09:45:46.100Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"878c94ab-c70e-4df5-9b2c-23f37444cf23","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:45:46.100Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"878c94ab-c70e-4df5-9b2c-23f37444cf23","name":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:46.100Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"878c94ab-c70e-4df5-9b2c-23f37444cf23","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:46.100Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"878c94ab-c70e-4df5-9b2c-23f37444cf23"}
{"level":"info","ts":"2026-06-05T09:45:46.103Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"878c94ab-c70e-4df5-9b2c-23f37444cf23","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:45:44Z"},"jobId":"fail-qjbjr","rayClusterName":"fail-78fz8","dashboardURL":"fail-78fz8-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-05T09:45:11Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:45:37Z","stateTransitionTimes":{"ready":"2026-06-05T09:45:37Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.23","serviceIP":"10.132.0.23","podName":"fail-78fz8-head-7jx8g","serviceName":"fail-78fz8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:45:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:45:37Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:45:44Z"},"jobId":"fail-qjbjr","rayClusterName":"fail-78fz8","dashboardURL":"fail-78fz8-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-05T09:45:11Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:45:37Z","stateTransitionTimes":{"ready":"2026-06-05T09:45:37Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.23","serviceIP":"10.132.0.23","podName":"fail-78fz8-head-7jx8g","serviceName":"fail-78fz8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:45:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:45:37Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:45:49.100Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"759e0b9b-9618-4a91-a464-79e3a4765df1","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:45:49.100Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"759e0b9b-9618-4a91-a464-79e3a4765df1","name":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:49.100Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"759e0b9b-9618-4a91-a464-79e3a4765df1","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:49.100Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"759e0b9b-9618-4a91-a464-79e3a4765df1"}
{"level":"info","ts":"2026-06-05T09:45:49.103Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"759e0b9b-9618-4a91-a464-79e3a4765df1","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:45:44Z"},"jobId":"fail-qjbjr","rayClusterName":"fail-78fz8","dashboardURL":"fail-78fz8-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-05T09:45:11Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:45:37Z","stateTransitionTimes":{"ready":"2026-06-05T09:45:37Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.23","serviceIP":"10.132.0.23","podName":"fail-78fz8-head-7jx8g","serviceName":"fail-78fz8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:45:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:45:37Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:45:44Z","endTime":"2026-06-05T09:45:46Z"},"jobId":"fail-qjbjr","rayClusterName":"fail-78fz8","dashboardURL":"fail-78fz8-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-05 09:45:44,989\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-qjbjr: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-05T09:45:11Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:45:37Z","stateTransitionTimes":{"ready":"2026-06-05T09:45:37Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.23","serviceIP":"10.132.0.23","podName":"fail-78fz8-head-7jx8g","serviceName":"fail-78fz8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:45:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:45:37Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:45:49.103Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"759e0b9b-9618-4a91-a464-79e3a4765df1","old JobStatus":"PENDING","new JobStatus":"FAILED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-05T09:45:49.119Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"6553e386-deb8-4bfa-8c67-ea3ef3a24b47","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:45:49.119Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"6553e386-deb8-4bfa-8c67-ea3ef3a24b47","name":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:49.119Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"6553e386-deb8-4bfa-8c67-ea3ef3a24b47","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:49.119Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"6553e386-deb8-4bfa-8c67-ea3ef3a24b47"}
{"level":"info","ts":"2026-06-05T09:45:49.122Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"6553e386-deb8-4bfa-8c67-ea3ef3a24b47","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:45:44Z","endTime":"2026-06-05T09:45:46Z"},"jobId":"fail-qjbjr","rayClusterName":"fail-78fz8","dashboardURL":"fail-78fz8-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-05 09:45:44,989\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-qjbjr: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-05T09:45:11Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:45:37Z","stateTransitionTimes":{"ready":"2026-06-05T09:45:37Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.23","serviceIP":"10.132.0.23","podName":"fail-78fz8-head-7jx8g","serviceName":"fail-78fz8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:45:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:45:37Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:45:44Z","endTime":"2026-06-05T09:45:46Z"},"jobId":"fail-qjbjr","rayClusterName":"fail-78fz8","dashboardURL":"fail-78fz8-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-05 09:45:44,989\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-qjbjr: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-05T09:45:11Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:45:37Z","stateTransitionTimes":{"ready":"2026-06-05T09:45:37Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.23","serviceIP":"10.132.0.23","podName":"fail-78fz8-head-7jx8g","serviceName":"fail-78fz8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:45:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:45:37Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:45:52.120Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"0437e570-f92a-439f-a60c-ac6f3e787c8b","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:45:52.120Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"0437e570-f92a-439f-a60c-ac6f3e787c8b","name":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:52.120Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"0437e570-f92a-439f-a60c-ac6f3e787c8b","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:52.120Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"0437e570-f92a-439f-a60c-ac6f3e787c8b"}
{"level":"info","ts":"2026-06-05T09:45:52.123Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"0437e570-f92a-439f-a60c-ac6f3e787c8b","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:45:44Z","endTime":"2026-06-05T09:45:46Z"},"jobId":"fail-qjbjr","rayClusterName":"fail-78fz8","dashboardURL":"fail-78fz8-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-05 09:45:44,989\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-qjbjr: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-05T09:45:11Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:45:37Z","stateTransitionTimes":{"ready":"2026-06-05T09:45:37Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.23","serviceIP":"10.132.0.23","podName":"fail-78fz8-head-7jx8g","serviceName":"fail-78fz8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:45:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:45:37Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:45:44Z","endTime":"2026-06-05T09:45:46Z"},"jobId":"fail-qjbjr","rayClusterName":"fail-78fz8","dashboardURL":"fail-78fz8-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-05 09:45:44,989\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-qjbjr: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-05T09:45:11Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:45:37Z","stateTransitionTimes":{"ready":"2026-06-05T09:45:37Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.23","serviceIP":"10.132.0.23","podName":"fail-78fz8-head-7jx8g","serviceName":"fail-78fz8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:45:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:45:37Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:45:55.124Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"4055233d-623d-449b-8c5a-9feea358346f","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:45:55.124Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"4055233d-623d-449b-8c5a-9feea358346f","name":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:55.124Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"4055233d-623d-449b-8c5a-9feea358346f","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:55.124Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"4055233d-623d-449b-8c5a-9feea358346f"}
{"level":"info","ts":"2026-06-05T09:45:55.127Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"4055233d-623d-449b-8c5a-9feea358346f","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:45:44Z","endTime":"2026-06-05T09:45:46Z"},"jobId":"fail-qjbjr","rayClusterName":"fail-78fz8","dashboardURL":"fail-78fz8-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-05 09:45:44,989\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-qjbjr: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-05T09:45:11Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:45:37Z","stateTransitionTimes":{"ready":"2026-06-05T09:45:37Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.23","serviceIP":"10.132.0.23","podName":"fail-78fz8-head-7jx8g","serviceName":"fail-78fz8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:45:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:45:37Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:45:44Z","endTime":"2026-06-05T09:45:46Z"},"jobId":"fail-qjbjr","rayClusterName":"fail-78fz8","dashboardURL":"fail-78fz8-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-05 09:45:44,989\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-qjbjr: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-05T09:45:11Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:45:37Z","stateTransitionTimes":{"ready":"2026-06-05T09:45:37Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.23","serviceIP":"10.132.0.23","podName":"fail-78fz8-head-7jx8g","serviceName":"fail-78fz8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:45:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:45:37Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:45:56.126Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"6f1f1ce1-5e46-4d9d-a0cc-f7b19df2d788","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:45:56.127Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"6f1f1ce1-5e46-4d9d-a0cc-f7b19df2d788","name":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:56.127Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"6f1f1ce1-5e46-4d9d-a0cc-f7b19df2d788","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:56.127Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"6f1f1ce1-5e46-4d9d-a0cc-f7b19df2d788"}
{"level":"info","ts":"2026-06-05T09:45:56.130Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"6f1f1ce1-5e46-4d9d-a0cc-f7b19df2d788","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:45:44Z","endTime":"2026-06-05T09:45:46Z"},"jobId":"fail-qjbjr","rayClusterName":"fail-78fz8","dashboardURL":"fail-78fz8-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-05 09:45:44,989\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-qjbjr: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-05T09:45:11Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:45:37Z","stateTransitionTimes":{"ready":"2026-06-05T09:45:37Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.23","serviceIP":"10.132.0.23","podName":"fail-78fz8-head-7jx8g","serviceName":"fail-78fz8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:45:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:45:37Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:45:44Z","endTime":"2026-06-05T09:45:46Z"},"jobId":"fail-qjbjr","rayClusterName":"fail-78fz8","dashboardURL":"fail-78fz8-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-05 09:45:44,989\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-qjbjr: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-05T09:45:11Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:45:37Z","stateTransitionTimes":{"ready":"2026-06-05T09:45:37Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.23","serviceIP":"10.132.0.23","podName":"fail-78fz8-head-7jx8g","serviceName":"fail-78fz8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:45:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:45:37Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:45:57.135Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"5579cddd-813b-4ce0-9a46-61e3a84987c7","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:45:57.135Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"5579cddd-813b-4ce0-9a46-61e3a84987c7","name":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:57.135Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"5579cddd-813b-4ce0-9a46-61e3a84987c7","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:57.135Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"5579cddd-813b-4ce0-9a46-61e3a84987c7"}
{"level":"info","ts":"2026-06-05T09:45:57.138Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"5579cddd-813b-4ce0-9a46-61e3a84987c7","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:45:44Z","endTime":"2026-06-05T09:45:46Z"},"jobId":"fail-qjbjr","rayClusterName":"fail-78fz8","dashboardURL":"fail-78fz8-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-05 09:45:44,989\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-qjbjr: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-05T09:45:11Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:45:37Z","stateTransitionTimes":{"ready":"2026-06-05T09:45:37Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.23","serviceIP":"10.132.0.23","podName":"fail-78fz8-head-7jx8g","serviceName":"fail-78fz8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:45:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:45:37Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:45:44Z","endTime":"2026-06-05T09:45:46Z"},"jobId":"fail-qjbjr","rayClusterName":"fail-78fz8","dashboardURL":"fail-78fz8-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-05 09:45:44,989\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-qjbjr: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-05T09:45:11Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:45:37Z","stateTransitionTimes":{"ready":"2026-06-05T09:45:37Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.23","serviceIP":"10.132.0.23","podName":"fail-78fz8-head-7jx8g","serviceName":"fail-78fz8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:45:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:45:37Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:45:57.152Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"6381f5a3-535f-4062-95b7-5e38463c8339","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:45:57.152Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"6381f5a3-535f-4062-95b7-5e38463c8339","name":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:57.152Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"6381f5a3-535f-4062-95b7-5e38463c8339","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:57.152Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"6381f5a3-535f-4062-95b7-5e38463c8339"}
{"level":"info","ts":"2026-06-05T09:45:57.155Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"6381f5a3-535f-4062-95b7-5e38463c8339","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:45:44Z","endTime":"2026-06-05T09:45:46Z"},"jobId":"fail-qjbjr","rayClusterName":"fail-78fz8","dashboardURL":"fail-78fz8-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-05 09:45:44,989\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-qjbjr: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-05T09:45:11Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:45:37Z","stateTransitionTimes":{"ready":"2026-06-05T09:45:37Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.23","serviceIP":"10.132.0.23","podName":"fail-78fz8-head-7jx8g","serviceName":"fail-78fz8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:45:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:45:37Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:45:44Z","endTime":"2026-06-05T09:45:46Z"},"jobId":"fail-qjbjr","rayClusterName":"fail-78fz8","dashboardURL":"fail-78fz8-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Failed","reason":"AppFailed","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-05 09:45:44,989\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-qjbjr: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-05T09:45:11Z","succeeded":0,"failed":1,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:45:37Z","stateTransitionTimes":{"ready":"2026-06-05T09:45:37Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.23","serviceIP":"10.132.0.23","podName":"fail-78fz8-head-7jx8g","serviceName":"fail-78fz8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:45:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:45:37Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:11Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:45:57.155Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"6381f5a3-535f-4062-95b7-5e38463c8339","old JobStatus":"FAILED","new JobStatus":"FAILED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Failed"}
{"level":"info","ts":"2026-06-05T09:45:57.170Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"ab9c6d32-7025-4d56-a781-7264f7cd8d7c","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:45:57.170Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"ab9c6d32-7025-4d56-a781-7264f7cd8d7c","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-05 09:45:57 +0000 UTC","Now":"2026-06-05T09:45:57.170Z","ShutdownTime":"2026-06-05T09:45:57.000Z"}
{"level":"info","ts":"2026-06-05T09:45:57.451Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"1d2925fd-e5f4-484d-ab8b-55eb24fcce47","DeletionTimestamp":"2026-06-05 09:45:57 +0000 UTC"}
{"level":"info","ts":"2026-06-05T09:45:57.451Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"1d2925fd-e5f4-484d-ab8b-55eb24fcce47","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-05T09:45:57.465Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"e3fa004c-1ad9-482e-8437-b67430ee0cba"}
{"level":"info","ts":"2026-06-05T09:45:57.478Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"05108f6d-fe94-4f82-815b-ecda249d102f"}
{"level":"info","ts":"2026-06-05T09:45:57.483Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"5d03fb50-5833-40a8-bcb9-3d2da26ff9e8","rayCluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:57.483Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"d271656d-2062-4f2b-93e6-e75052d10d4e"}
{"level":"info","ts":"2026-06-05T09:45:57.483Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"71a3b565-ec7d-4e23-a569-eacd6cca6e15","namespacedName":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:57.483Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"71a3b565-ec7d-4e23-a569-eacd6cca6e15","cluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:57.483Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"71a3b565-ec7d-4e23-a569-eacd6cca6e15","cluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:57.483Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"eb007f6b-ae33-4a8d-b646-808cc024a8a1"}
{"level":"info","ts":"2026-06-05T09:45:57.483Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"5d03fb50-5833-40a8-bcb9-3d2da26ff9e8","pod":"fail-78fz8-head-7jx8g","ip":"10.132.0.23"}
{"level":"info","ts":"2026-06-05T09:45:57.483Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"5d03fb50-5833-40a8-bcb9-3d2da26ff9e8","pod":"fail-78fz8-small-group-worker-l8sb8","ip":"10.133.0.45"}
{"level":"info","ts":"2026-06-05T09:45:57.483Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"71a3b565-ec7d-4e23-a569-eacd6cca6e15","namespace":"test-ns-thrrw","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-05T09:45:57.483Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"reconcileID":"09984bbb-d3de-4b7c-8d18-75273b3d5153"}
{"level":"info","ts":"2026-06-05T09:45:57.483Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"5d03fb50-5833-40a8-bcb9-3d2da26ff9e8","rayCluster":"fail-78fz8","podIPs":["10.132.0.23","10.133.0.45"]}
{"level":"info","ts":"2026-06-05T09:45:57.492Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"71a3b565-ec7d-4e23-a569-eacd6cca6e15","name":"kuberay-gateway-access","namespace":"test-ns-thrrw"}
{"level":"info","ts":"2026-06-05T09:45:57.497Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"71a3b565-ec7d-4e23-a569-eacd6cca6e15","name":"test-ns-thrrw-fail-78fz8","namespace":"opendatahub"}
{"level":"info","ts":"2026-06-05T09:45:57.504Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"71a3b565-ec7d-4e23-a569-eacd6cca6e15","configMap":"kube-rbac-proxy-config-fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:57.509Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"71a3b565-ec7d-4e23-a569-eacd6cca6e15","serviceAccount":"fail-78fz8-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:45:57.517Z","logger":"raycluster-resource","msg":"default","name":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:57.517Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-78fz8","namespace":"test-ns-thrrw"}
{"level":"info","ts":"2026-06-05T09:45:57.527Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"71a3b565-ec7d-4e23-a569-eacd6cca6e15","cluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:57.527Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"4d4d4e14-c59a-445b-a909-9aad633feeed","namespacedName":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:57.527Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"4d4d4e14-c59a-445b-a909-9aad633feeed","cluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:57.527Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"4d4d4e14-c59a-445b-a909-9aad633feeed","cluster":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:57.527Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"4d4d4e14-c59a-445b-a909-9aad633feeed","namespace":"test-ns-thrrw","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-05T09:45:57.527Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"4d4d4e14-c59a-445b-a909-9aad633feeed","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-06-05T09:45:57.527Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"54421d7c-a06c-4f64-adbf-035f885f0bfc"}
{"level":"info","ts":"2026-06-05T09:45:57.527Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"f6bcdb1e-218c-4b9e-802e-47d7b938e0f8"}
{"level":"info","ts":"2026-06-05T09:45:57.527Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"4c3ad91f-d473-4a09-b870-0b3cc306f6dd","namespace":"test-ns-thrrw","clusterName":"fail-78fz8"}
{"level":"error","ts":"2026-06-05T09:45:57.531Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"4d4d4e14-c59a-445b-a909-9aad633feeed","error":"rayclusters.ray.io \"fail-78fz8\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:214\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:155\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:57.531Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"4d4d4e14-c59a-445b-a909-9aad633feeed"}
{"level":"error","ts":"2026-06-05T09:45:57.531Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"4d4d4e14-c59a-445b-a909-9aad633feeed","error":"rayclusters.ray.io \"fail-78fz8\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:57.531Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"d328c01a-6964-48b4-bc59-60ff23a92a63","namespacedName":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:57.531Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"d328c01a-6964-48b4-bc59-60ff23a92a63"}
{"level":"info","ts":"2026-06-05T09:45:57.531Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"d328c01a-6964-48b4-bc59-60ff23a92a63","namespace":"test-ns-thrrw","authClusters":0}
{"level":"info","ts":"2026-06-05T09:45:57.531Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"d328c01a-6964-48b4-bc59-60ff23a92a63","namespace":"test-ns-thrrw"}
{"level":"info","ts":"2026-06-05T09:45:57.534Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"597a3cb2-b457-43bc-aa69-58b926515663"}
{"level":"info","ts":"2026-06-05T09:45:57.535Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"e0407df0-dc6f-4231-b0be-70435a801d05"}
{"level":"info","ts":"2026-06-05T09:45:57.536Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"e13872f1-191b-4685-992d-d67868135725","namespacedName":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:57.536Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"e13872f1-191b-4685-992d-d67868135725"}
{"level":"info","ts":"2026-06-05T09:45:57.536Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"e13872f1-191b-4685-992d-d67868135725","namespace":"test-ns-thrrw","authClusters":0}
{"level":"info","ts":"2026-06-05T09:45:57.536Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"e13872f1-191b-4685-992d-d67868135725","namespace":"test-ns-thrrw"}
{"level":"info","ts":"2026-06-05T09:45:57.542Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"7c685135-7673-420a-b026-2c2f267e58cd","namespacedName":{"name":"fail-78fz8","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:57.542Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"7c685135-7673-420a-b026-2c2f267e58cd"}
{"level":"info","ts":"2026-06-05T09:45:57.542Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"7c685135-7673-420a-b026-2c2f267e58cd","namespace":"test-ns-thrrw","authClusters":0}
{"level":"info","ts":"2026-06-05T09:45:57.542Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"7c685135-7673-420a-b026-2c2f267e58cd","namespace":"test-ns-thrrw"}
{"level":"error","ts":"2026-06-05T09:45:57.542Z","msg":"Failed to delete worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"4c3ad91f-d473-4a09-b870-0b3cc306f6dd","certificate":"ray-worker-cert-fail-78fz8","error":"certificates.cert-manager.io \"ray-worker-cert-fail-78fz8\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:268\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:57.617Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"4c3ad91f-d473-4a09-b870-0b3cc306f6dd","errorCount":1}
{"level":"info","ts":"2026-06-05T09:45:57.617Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"4c3ad91f-d473-4a09-b870-0b3cc306f6dd"}
{"level":"error","ts":"2026-06-05T09:45:57.617Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"4c3ad91f-d473-4a09-b870-0b3cc306f6dd","error":"certificates.cert-manager.io \"ray-worker-cert-fail-78fz8\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:57.622Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"f57c9cab-1363-4a8b-bc9f-9458d3cdcd4d","namespace":"test-ns-thrrw","clusterName":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:57.622Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-78fz8","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-78fz8","reconcileID":"f57c9cab-1363-4a8b-bc9f-9458d3cdcd4d","namespace":"test-ns-thrrw","clusterName":"fail-78fz8"}
{"level":"info","ts":"2026-06-05T09:45:58.128Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail","namespace":"test-ns-thrrw"},"reconcileID":"dd5e2324-0e76-4720-8fc8-5b967d582fca"}
{"level":"info","ts":"2026-06-05T09:45:58.473Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"31ac6f0d-b016-4727-9eae-a9e0e017bfc4","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:45:58.473Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"31ac6f0d-b016-4727-9eae-a9e0e017bfc4","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-05T09:45:58.485Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"31ac6f0d-b016-4727-9eae-a9e0e017bfc4"}
{"level":"info","ts":"2026-06-05T09:45:58.485Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"31ac6f0d-b016-4727-9eae-a9e0e017bfc4","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-05T09:45:58.485Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"31ac6f0d-b016-4727-9eae-a9e0e017bfc4","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-7swsf","rayClusterName":"fail-k8s-job-554g7","jobDeploymentStatus":"Initializing","startTime":"2026-06-05T09:45:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:45:58.485Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"31ac6f0d-b016-4727-9eae-a9e0e017bfc4","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-05T09:45:58.500Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"7cd193ea-f063-411d-b797-d2cb10c02aba","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:45:58.500Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"7cd193ea-f063-411d-b797-d2cb10c02aba","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:45:58.500Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"7cd193ea-f063-411d-b797-d2cb10c02aba","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:58.500Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"7cd193ea-f063-411d-b797-d2cb10c02aba","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:58.507Z","logger":"raycluster-resource","msg":"default","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:45:58.507Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}
{"level":"info","ts":"2026-06-05T09:45:58.527Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"f8cdaffe-6336-4038-a198-b068f438a645","rayCluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:45:58.527Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"f8cdaffe-6336-4038-a198-b068f438a645","rayCluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:45:58.527Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"38f4a60b-6e07-43a7-be5c-4ebd0f11a176","namespacedName":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:58.527Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"9f859197-b4b1-4417-9161-862f3b3299fd"}
{"level":"info","ts":"2026-06-05T09:45:58.527Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"9f859197-b4b1-4417-9161-862f3b3299fd"}
{"level":"info","ts":"2026-06-05T09:45:58.527Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"7aff1948-a982-4f78-806a-fcc5017666a6","cluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:45:58.527Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"38f4a60b-6e07-43a7-be5c-4ebd0f11a176","mode":"IntegratedOAuth","cluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:45:58.527Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"38f4a60b-6e07-43a7-be5c-4ebd0f11a176","cluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:45:58.527Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"38f4a60b-6e07-43a7-be5c-4ebd0f11a176","cluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:45:58.528Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"7cd193ea-f063-411d-b797-d2cb10c02aba","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:58.528Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"7cd193ea-f063-411d-b797-d2cb10c02aba"}
{"level":"info","ts":"2026-06-05T09:45:58.528Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"7cd193ea-f063-411d-b797-d2cb10c02aba","RayCluster":"fail-k8s-job-554g7","State":""}
{"level":"info","ts":"2026-06-05T09:45:58.528Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"e35ffa4a-9106-449b-8388-8c47ac74921f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:45:58.528Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"e35ffa4a-9106-449b-8388-8c47ac74921f","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:45:58.528Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"e35ffa4a-9106-449b-8388-8c47ac74921f","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:58.528Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"e35ffa4a-9106-449b-8388-8c47ac74921f"}
{"level":"info","ts":"2026-06-05T09:45:58.528Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"e35ffa4a-9106-449b-8388-8c47ac74921f","RayCluster":"fail-k8s-job-554g7","State":""}
{"level":"info","ts":"2026-06-05T09:45:58.539Z","logger":"raycluster-resource","msg":"default","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:45:58.539Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}
{"level":"info","ts":"2026-06-05T09:45:58.545Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"9f859197-b4b1-4417-9161-862f3b3299fd","name":"fail-k8s-job-554g7-head-svc"}
{"level":"info","ts":"2026-06-05T09:45:58.545Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"9f859197-b4b1-4417-9161-862f3b3299fd","error":"head certificate secret ray-head-secret-fail-k8s-job-554g7 not found"}
{"level":"error","ts":"2026-06-05T09:45:58.545Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"9f859197-b4b1-4417-9161-862f3b3299fd","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-554g7 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:58.545Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"9f859197-b4b1-4417-9161-862f3b3299fd","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-554g7 not found"}
{"level":"info","ts":"2026-06-05T09:45:58.545Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"9f859197-b4b1-4417-9161-862f3b3299fd","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-k8s-job-554g7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:58.545Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"9f859197-b4b1-4417-9161-862f3b3299fd","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-k8s-job-554g7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:58.545Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"9f859197-b4b1-4417-9161-862f3b3299fd","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-k8s-job-554g7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:58.545Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"9f859197-b4b1-4417-9161-862f3b3299fd","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-05T09:45:58.545Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"9f859197-b4b1-4417-9161-862f3b3299fd","name":"fail-k8s-job-554g7","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:45:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-k8s-job-554g7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:45:58.546Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"f8cdaffe-6336-4038-a198-b068f438a645","rayCluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:45:58.546Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"f8cdaffe-6336-4038-a198-b068f438a645","rayCluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:45:58.553Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"aac4f4bf-6f72-42eb-a8dc-a2174d8ebfa1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:45:58.553Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"aac4f4bf-6f72-42eb-a8dc-a2174d8ebfa1","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:45:58.553Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"aac4f4bf-6f72-42eb-a8dc-a2174d8ebfa1","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:58.558Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"7aff1948-a982-4f78-806a-fcc5017666a6","cluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:45:58.558Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"f56a871c-27e1-4bbf-b777-2b82f488f078","cluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:45:58.564Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"9f859197-b4b1-4417-9161-862f3b3299fd","name":"fail-k8s-job-554g7","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-554g7\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7"}}
{"level":"info","ts":"2026-06-05T09:45:58.564Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"9f859197-b4b1-4417-9161-862f3b3299fd"}
{"level":"error","ts":"2026-06-05T09:45:58.564Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"9f859197-b4b1-4417-9161-862f3b3299fd","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-554g7 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:58.605Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"19bde663-91e4-4c4a-a0a3-a40ec156b3c2"}
{"level":"info","ts":"2026-06-05T09:45:58.605Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"19bde663-91e4-4c4a-a0a3-a40ec156b3c2"}
{"level":"info","ts":"2026-06-05T09:45:58.605Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"19bde663-91e4-4c4a-a0a3-a40ec156b3c2","1 head service found":"fail-k8s-job-554g7-head-svc"}
{"level":"info","ts":"2026-06-05T09:45:58.605Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"38f4a60b-6e07-43a7-be5c-4ebd0f11a176","name":"fail-k8s-job-554g7-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:45:58.606Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"19bde663-91e4-4c4a-a0a3-a40ec156b3c2","error":"head certificate secret ray-head-secret-fail-k8s-job-554g7 not found"}
{"level":"info","ts":"2026-06-05T09:45:58.606Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"f8cdaffe-6336-4038-a198-b068f438a645"}
{"level":"error","ts":"2026-06-05T09:45:58.606Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"19bde663-91e4-4c4a-a0a3-a40ec156b3c2","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-554g7 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:58.606Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"19bde663-91e4-4c4a-a0a3-a40ec156b3c2","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-554g7 not found"}
{"level":"info","ts":"2026-06-05T09:45:58.606Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"19bde663-91e4-4c4a-a0a3-a40ec156b3c2","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-k8s-job-554g7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:58.605Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"aac4f4bf-6f72-42eb-a8dc-a2174d8ebfa1"}
{"level":"info","ts":"2026-06-05T09:45:58.606Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"aac4f4bf-6f72-42eb-a8dc-a2174d8ebfa1","RayCluster":"fail-k8s-job-554g7","State":""}
{"level":"info","ts":"2026-06-05T09:45:58.606Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"19bde663-91e4-4c4a-a0a3-a40ec156b3c2","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-k8s-job-554g7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:58.606Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"f8cdaffe-6336-4038-a198-b068f438a645","rayCluster":"fail-k8s-job-554g7","secret":"ca-secret"}
{"level":"info","ts":"2026-06-05T09:45:58.606Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"19bde663-91e4-4c4a-a0a3-a40ec156b3c2","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-k8s-job-554g7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:58.606Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"19bde663-91e4-4c4a-a0a3-a40ec156b3c2","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-05T09:45:58.606Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"19bde663-91e4-4c4a-a0a3-a40ec156b3c2","name":"fail-k8s-job-554g7","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:45:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-k8s-job-554g7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:45:58.606Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"f8cdaffe-6336-4038-a198-b068f438a645","rayCluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:45:58.611Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"f56a871c-27e1-4bbf-b777-2b82f488f078","name":"fail-k8s-job-554g7-head"}
{"level":"info","ts":"2026-06-05T09:45:58.612Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"38f4a60b-6e07-43a7-be5c-4ebd0f11a176","name":"kuberay-gateway-access","namespace":"test-ns-thrrw","operation":"created"}
{"level":"info","ts":"2026-06-05T09:45:58.617Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"19bde663-91e4-4c4a-a0a3-a40ec156b3c2"}
{"level":"error","ts":"2026-06-05T09:45:58.617Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"19bde663-91e4-4c4a-a0a3-a40ec156b3c2","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-554g7 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:58.618Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"fd4c243b-1de7-46b0-aee4-8bfe1a0f017b"}
{"level":"info","ts":"2026-06-05T09:45:58.618Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"fd4c243b-1de7-46b0-aee4-8bfe1a0f017b"}
{"level":"info","ts":"2026-06-05T09:45:58.618Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"fd4c243b-1de7-46b0-aee4-8bfe1a0f017b","1 head service found":"fail-k8s-job-554g7-head-svc"}
{"level":"info","ts":"2026-06-05T09:45:58.618Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"fd4c243b-1de7-46b0-aee4-8bfe1a0f017b","error":"head certificate secret ray-head-secret-fail-k8s-job-554g7 not found"}
{"level":"info","ts":"2026-06-05T09:45:58.618Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"a95ab96a-dccd-4b7c-8b64-838fd2591a50","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:45:58.618Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"a95ab96a-dccd-4b7c-8b64-838fd2591a50","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:45:58.618Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"a95ab96a-dccd-4b7c-8b64-838fd2591a50","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:58.618Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"a95ab96a-dccd-4b7c-8b64-838fd2591a50"}
{"level":"info","ts":"2026-06-05T09:45:58.618Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"a95ab96a-dccd-4b7c-8b64-838fd2591a50","RayCluster":"fail-k8s-job-554g7","State":""}
{"level":"error","ts":"2026-06-05T09:45:58.618Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"fd4c243b-1de7-46b0-aee4-8bfe1a0f017b","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-554g7 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:58.618Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"fd4c243b-1de7-46b0-aee4-8bfe1a0f017b","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-554g7 not found"}
{"level":"info","ts":"2026-06-05T09:45:58.618Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"fd4c243b-1de7-46b0-aee4-8bfe1a0f017b","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-k8s-job-554g7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:58.618Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"fd4c243b-1de7-46b0-aee4-8bfe1a0f017b","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-k8s-job-554g7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:58.618Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"fd4c243b-1de7-46b0-aee4-8bfe1a0f017b","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-k8s-job-554g7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:58.618Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"fd4c243b-1de7-46b0-aee4-8bfe1a0f017b"}
{"level":"error","ts":"2026-06-05T09:45:58.618Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"fd4c243b-1de7-46b0-aee4-8bfe1a0f017b","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-554g7 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:58.619Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"f8cdaffe-6336-4038-a198-b068f438a645","rayCluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:45:58.622Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"38f4a60b-6e07-43a7-be5c-4ebd0f11a176","name":"test-ns-thrrw-fail-k8s-job-554g7","operation":"created"}
{"level":"info","ts":"2026-06-05T09:45:58.625Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"f56a871c-27e1-4bbf-b777-2b82f488f078","name":"fail-k8s-job-554g7-workers"}
{"level":"info","ts":"2026-06-05T09:45:58.625Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"f56a871c-27e1-4bbf-b777-2b82f488f078","cluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:45:58.625Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"6c1642e5-7e0b-443b-83b9-b5e3efa331cd","cluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:45:58.627Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"38f4a60b-6e07-43a7-be5c-4ebd0f11a176","name":"kube-rbac-proxy-config-fail-k8s-job-554g7","operation":"created"}
{"level":"info","ts":"2026-06-05T09:45:58.705Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"3a9e77e3-afdf-4d89-b93a-31d00c229f64"}
{"level":"info","ts":"2026-06-05T09:45:58.706Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"3a9e77e3-afdf-4d89-b93a-31d00c229f64"}
{"level":"info","ts":"2026-06-05T09:45:58.706Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"3a9e77e3-afdf-4d89-b93a-31d00c229f64","1 head service found":"fail-k8s-job-554g7-head-svc"}
{"level":"info","ts":"2026-06-05T09:45:58.706Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"3a9e77e3-afdf-4d89-b93a-31d00c229f64","error":"head certificate secret ray-head-secret-fail-k8s-job-554g7 not found"}
{"level":"error","ts":"2026-06-05T09:45:58.706Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"3a9e77e3-afdf-4d89-b93a-31d00c229f64","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-554g7 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:58.706Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"3a9e77e3-afdf-4d89-b93a-31d00c229f64","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-554g7 not found"}
{"level":"error","ts":"2026-06-05T09:45:58.706Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"38f4a60b-6e07-43a7-be5c-4ebd0f11a176","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-554g7\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:58.706Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"38f4a60b-6e07-43a7-be5c-4ebd0f11a176"}
{"level":"error","ts":"2026-06-05T09:45:58.706Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"38f4a60b-6e07-43a7-be5c-4ebd0f11a176","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-554g7\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:58.706Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"3a9e77e3-afdf-4d89-b93a-31d00c229f64","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-k8s-job-554g7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:58.706Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"74e6c56a-2e32-45c9-a4d3-d097797c8426","namespacedName":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:58.706Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"3a9e77e3-afdf-4d89-b93a-31d00c229f64","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-k8s-job-554g7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:58.706Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"3a9e77e3-afdf-4d89-b93a-31d00c229f64","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-k8s-job-554g7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:58.706Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"3a9e77e3-afdf-4d89-b93a-31d00c229f64"}
{"level":"error","ts":"2026-06-05T09:45:58.706Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"3a9e77e3-afdf-4d89-b93a-31d00c229f64","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-554g7 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:58.706Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"f8cdaffe-6336-4038-a198-b068f438a645"}
{"level":"info","ts":"2026-06-05T09:45:58.706Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"74e6c56a-2e32-45c9-a4d3-d097797c8426","mode":"IntegratedOAuth","cluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:45:58.706Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"74e6c56a-2e32-45c9-a4d3-d097797c8426","cluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:45:58.706Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"f8cdaffe-6336-4038-a198-b068f438a645","rayCluster":"fail-k8s-job-554g7","podIPs":[]}
{"level":"info","ts":"2026-06-05T09:45:58.711Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"6c1642e5-7e0b-443b-83b9-b5e3efa331cd","name":"fail-k8s-job-554g7-head"}
{"level":"info","ts":"2026-06-05T09:45:58.718Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"74e6c56a-2e32-45c9-a4d3-d097797c8426","cluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:45:58.718Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"9c9da1bf-9ba6-44c7-9325-ef6b32f32dba","namespacedName":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:58.718Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"9c9da1bf-9ba6-44c7-9325-ef6b32f32dba","mode":"IntegratedOAuth","cluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:45:58.718Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"9c9da1bf-9ba6-44c7-9325-ef6b32f32dba","cluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:45:58.718Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"a7f40521-a3fe-4017-a47c-9b478b5e3402","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:45:58.718Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"a7f40521-a3fe-4017-a47c-9b478b5e3402","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:45:58.718Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"a7f40521-a3fe-4017-a47c-9b478b5e3402","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:45:58.719Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"a7f40521-a3fe-4017-a47c-9b478b5e3402"}
{"level":"info","ts":"2026-06-05T09:45:58.719Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"a7f40521-a3fe-4017-a47c-9b478b5e3402","RayCluster":"fail-k8s-job-554g7","State":""}
{"level":"info","ts":"2026-06-05T09:45:58.720Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"f8cdaffe-6336-4038-a198-b068f438a645"}
{"level":"info","ts":"2026-06-05T09:45:58.720Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"f8cdaffe-6336-4038-a198-b068f438a645","rayCluster":"fail-k8s-job-554g7","podIPs":[]}
{"level":"info","ts":"2026-06-05T09:45:58.720Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"f8cdaffe-6336-4038-a198-b068f438a645"}
{"level":"info","ts":"2026-06-05T09:45:58.720Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"6ebcf5b6-6dee-4c19-abb9-f82403395a25","rayCluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:45:58.720Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"6ebcf5b6-6dee-4c19-abb9-f82403395a25"}
{"level":"info","ts":"2026-06-05T09:45:58.724Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"6c1642e5-7e0b-443b-83b9-b5e3efa331cd","name":"fail-k8s-job-554g7-workers"}
{"level":"info","ts":"2026-06-05T09:45:58.724Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"6c1642e5-7e0b-443b-83b9-b5e3efa331cd","cluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:45:58.724Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"cf15bc44-601e-4ed4-84d3-b97e335d20e7","cluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:45:58.726Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"9c9da1bf-9ba6-44c7-9325-ef6b32f32dba","cluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:45:58.805Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"cf15bc44-601e-4ed4-84d3-b97e335d20e7","name":"fail-k8s-job-554g7-head"}
{"level":"info","ts":"2026-06-05T09:45:58.806Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"da28329e-82f5-4741-b6e9-2aa0fab3c775"}
{"level":"info","ts":"2026-06-05T09:45:58.806Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"da28329e-82f5-4741-b6e9-2aa0fab3c775"}
{"level":"info","ts":"2026-06-05T09:45:58.806Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"da28329e-82f5-4741-b6e9-2aa0fab3c775","1 head service found":"fail-k8s-job-554g7-head-svc"}
{"level":"info","ts":"2026-06-05T09:45:58.806Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"da28329e-82f5-4741-b6e9-2aa0fab3c775","error":"head certificate secret ray-head-secret-fail-k8s-job-554g7 not found"}
{"level":"error","ts":"2026-06-05T09:45:58.806Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"da28329e-82f5-4741-b6e9-2aa0fab3c775","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-554g7 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:58.806Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"da28329e-82f5-4741-b6e9-2aa0fab3c775","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-554g7 not found"}
{"level":"info","ts":"2026-06-05T09:45:58.806Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"da28329e-82f5-4741-b6e9-2aa0fab3c775","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-k8s-job-554g7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:58.806Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"da28329e-82f5-4741-b6e9-2aa0fab3c775","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-k8s-job-554g7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:58.806Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"da28329e-82f5-4741-b6e9-2aa0fab3c775","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-k8s-job-554g7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:58.806Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"da28329e-82f5-4741-b6e9-2aa0fab3c775"}
{"level":"error","ts":"2026-06-05T09:45:58.806Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"da28329e-82f5-4741-b6e9-2aa0fab3c775","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-554g7 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:58.817Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"cf15bc44-601e-4ed4-84d3-b97e335d20e7","name":"fail-k8s-job-554g7-workers"}
{"level":"info","ts":"2026-06-05T09:45:58.817Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"cf15bc44-601e-4ed4-84d3-b97e335d20e7","cluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:45:58.886Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"8e41d110-3021-46c8-a1ed-119d9ea8b15d"}
{"level":"info","ts":"2026-06-05T09:45:58.886Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"8e41d110-3021-46c8-a1ed-119d9ea8b15d"}
{"level":"info","ts":"2026-06-05T09:45:58.886Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"8e41d110-3021-46c8-a1ed-119d9ea8b15d","1 head service found":"fail-k8s-job-554g7-head-svc"}
{"level":"info","ts":"2026-06-05T09:45:58.886Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"8e41d110-3021-46c8-a1ed-119d9ea8b15d","error":"head certificate secret ray-head-secret-fail-k8s-job-554g7 not found"}
{"level":"error","ts":"2026-06-05T09:45:58.886Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"8e41d110-3021-46c8-a1ed-119d9ea8b15d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-554g7 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:58.887Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"8e41d110-3021-46c8-a1ed-119d9ea8b15d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-554g7 not found"}
{"level":"info","ts":"2026-06-05T09:45:58.887Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"8e41d110-3021-46c8-a1ed-119d9ea8b15d","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-k8s-job-554g7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:58.887Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"8e41d110-3021-46c8-a1ed-119d9ea8b15d","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-k8s-job-554g7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:58.887Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"8e41d110-3021-46c8-a1ed-119d9ea8b15d","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-k8s-job-554g7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:58.887Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"8e41d110-3021-46c8-a1ed-119d9ea8b15d"}
{"level":"error","ts":"2026-06-05T09:45:58.887Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"8e41d110-3021-46c8-a1ed-119d9ea8b15d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-554g7 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:59.047Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"01159774-6a31-44f0-842c-116d39884512"}
{"level":"info","ts":"2026-06-05T09:45:59.047Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"01159774-6a31-44f0-842c-116d39884512"}
{"level":"info","ts":"2026-06-05T09:45:59.047Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"01159774-6a31-44f0-842c-116d39884512","1 head service found":"fail-k8s-job-554g7-head-svc"}
{"level":"info","ts":"2026-06-05T09:45:59.047Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"01159774-6a31-44f0-842c-116d39884512","error":"head certificate secret ray-head-secret-fail-k8s-job-554g7 not found"}
{"level":"error","ts":"2026-06-05T09:45:59.047Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"01159774-6a31-44f0-842c-116d39884512","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-554g7 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:59.047Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"01159774-6a31-44f0-842c-116d39884512","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-554g7 not found"}
{"level":"info","ts":"2026-06-05T09:45:59.047Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"01159774-6a31-44f0-842c-116d39884512","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-k8s-job-554g7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:59.047Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"01159774-6a31-44f0-842c-116d39884512","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-k8s-job-554g7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:59.047Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"01159774-6a31-44f0-842c-116d39884512","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-k8s-job-554g7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:59.048Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"01159774-6a31-44f0-842c-116d39884512"}
{"level":"error","ts":"2026-06-05T09:45:59.048Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"01159774-6a31-44f0-842c-116d39884512","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-554g7 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:59.369Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d54636fb-e791-41a7-8583-c012e16f412d"}
{"level":"info","ts":"2026-06-05T09:45:59.369Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d54636fb-e791-41a7-8583-c012e16f412d"}
{"level":"info","ts":"2026-06-05T09:45:59.369Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d54636fb-e791-41a7-8583-c012e16f412d","1 head service found":"fail-k8s-job-554g7-head-svc"}
{"level":"info","ts":"2026-06-05T09:45:59.369Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d54636fb-e791-41a7-8583-c012e16f412d","error":"head certificate secret ray-head-secret-fail-k8s-job-554g7 not found"}
{"level":"error","ts":"2026-06-05T09:45:59.369Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d54636fb-e791-41a7-8583-c012e16f412d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-554g7 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:45:59.369Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d54636fb-e791-41a7-8583-c012e16f412d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-554g7 not found"}
{"level":"info","ts":"2026-06-05T09:45:59.369Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d54636fb-e791-41a7-8583-c012e16f412d","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-k8s-job-554g7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:59.369Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d54636fb-e791-41a7-8583-c012e16f412d","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-k8s-job-554g7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:59.369Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d54636fb-e791-41a7-8583-c012e16f412d","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-k8s-job-554g7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:45:59.369Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d54636fb-e791-41a7-8583-c012e16f412d"}
{"level":"error","ts":"2026-06-05T09:45:59.369Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d54636fb-e791-41a7-8583-c012e16f412d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-554g7 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:46:00.010Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"df602c87-c9d0-4f6b-a044-6b4b908559ea"}
{"level":"info","ts":"2026-06-05T09:46:00.010Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"df602c87-c9d0-4f6b-a044-6b4b908559ea"}
{"level":"info","ts":"2026-06-05T09:46:00.010Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"df602c87-c9d0-4f6b-a044-6b4b908559ea","1 head service found":"fail-k8s-job-554g7-head-svc"}
{"level":"info","ts":"2026-06-05T09:46:00.010Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"df602c87-c9d0-4f6b-a044-6b4b908559ea","error":"head certificate secret ray-head-secret-fail-k8s-job-554g7 not found"}
{"level":"error","ts":"2026-06-05T09:46:00.010Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"df602c87-c9d0-4f6b-a044-6b4b908559ea","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-554g7 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:46:00.010Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"df602c87-c9d0-4f6b-a044-6b4b908559ea","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-554g7 not found"}
{"level":"info","ts":"2026-06-05T09:46:00.010Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"df602c87-c9d0-4f6b-a044-6b4b908559ea","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-k8s-job-554g7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:46:00.010Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"df602c87-c9d0-4f6b-a044-6b4b908559ea","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-k8s-job-554g7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:46:00.010Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"df602c87-c9d0-4f6b-a044-6b4b908559ea","filter labels":["test-ns-thrrw",{"ray.io/cluster":"fail-k8s-job-554g7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:46:00.010Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"df602c87-c9d0-4f6b-a044-6b4b908559ea"}
{"level":"error","ts":"2026-06-05T09:46:00.010Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"df602c87-c9d0-4f6b-a044-6b4b908559ea","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-554g7 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:46:01.291Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d4268561-0b4f-4a2b-95a5-5baf0bebe1e8"}
{"level":"info","ts":"2026-06-05T09:46:01.291Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d4268561-0b4f-4a2b-95a5-5baf0bebe1e8"}
{"level":"info","ts":"2026-06-05T09:46:01.291Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d4268561-0b4f-4a2b-95a5-5baf0bebe1e8","1 head service found":"fail-k8s-job-554g7-head-svc"}
{"level":"info","ts":"2026-06-05T09:46:01.291Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d4268561-0b4f-4a2b-95a5-5baf0bebe1e8"}
{"level":"info","ts":"2026-06-05T09:46:01.291Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d4268561-0b4f-4a2b-95a5-5baf0bebe1e8","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:46:01.291Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d4268561-0b4f-4a2b-95a5-5baf0bebe1e8"}
{"level":"info","ts":"2026-06-05T09:46:01.291Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d4268561-0b4f-4a2b-95a5-5baf0bebe1e8","mode":"IntegratedOAuth","cluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:46:01.291Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d4268561-0b4f-4a2b-95a5-5baf0bebe1e8","cluster":"fail-k8s-job-554g7","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:46:01.291Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d4268561-0b4f-4a2b-95a5-5baf0bebe1e8","cluster":"fail-k8s-job-554g7","authType":"IntegratedOAuth","serviceAccount":"fail-k8s-job-554g7-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-05T09:46:01.291Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d4268561-0b4f-4a2b-95a5-5baf0bebe1e8","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"fail-k8s-job-554g7","ray.io/group":"headgroup","ray.io/identifier":"fail-k8s-job-554g7-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-05T09:46:01.291Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d4268561-0b4f-4a2b-95a5-5baf0bebe1e8","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"fail-k8s-job-554g7-head-svc.test-ns-thrrw.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-05T09:46:01.291Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d4268561-0b4f-4a2b-95a5-5baf0bebe1e8","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-k8s-job-554g7-head-svc.test-ns-thrrw.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-05T09:46:01.291Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d4268561-0b4f-4a2b-95a5-5baf0bebe1e8","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-k8s-job-554g7-head-svc.test-ns-thrrw.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-05T09:46:01.291Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d4268561-0b4f-4a2b-95a5-5baf0bebe1e8","enabled":true}
{"level":"info","ts":"2026-06-05T09:46:01.309Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d4268561-0b4f-4a2b-95a5-5baf0bebe1e8","name":"fail-k8s-job-554g7-head-wt9l4"}
{"level":"info","ts":"2026-06-05T09:46:01.309Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d4268561-0b4f-4a2b-95a5-5baf0bebe1e8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:46:01.309Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d4268561-0b4f-4a2b-95a5-5baf0bebe1e8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:46:01.309Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d4268561-0b4f-4a2b-95a5-5baf0bebe1e8","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-05T09:46:01.309Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d4268561-0b4f-4a2b-95a5-5baf0bebe1e8","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-05T09:46:01.309Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d4268561-0b4f-4a2b-95a5-5baf0bebe1e8","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-05T09:46:01.309Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d4268561-0b4f-4a2b-95a5-5baf0bebe1e8"}
{"level":"info","ts":"2026-06-05T09:46:01.309Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d4268561-0b4f-4a2b-95a5-5baf0bebe1e8","nodeType":"worker","rayStartParams":{"address":"fail-k8s-job-554g7-head-svc.test-ns-thrrw.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-05T09:46:01.310Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d4268561-0b4f-4a2b-95a5-5baf0bebe1e8","rayStartCmd":"ray start  --address=fail-k8s-job-554g7-head-svc.test-ns-thrrw.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:46:01.310Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d4268561-0b4f-4a2b-95a5-5baf0bebe1e8","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-k8s-job-554g7-head-svc.test-ns-thrrw.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:46:01.310Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d4268561-0b4f-4a2b-95a5-5baf0bebe1e8","enabled":true}
{"level":"info","ts":"2026-06-05T09:46:01.324Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d4268561-0b4f-4a2b-95a5-5baf0bebe1e8","name":"fail-k8s-job-554g7-small-group-worker-svfxq"}
{"level":"info","ts":"2026-06-05T09:46:01.324Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d4268561-0b4f-4a2b-95a5-5baf0bebe1e8","podName":"fail-k8s-job-554g7-head-wt9l4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:46:01.324Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d4268561-0b4f-4a2b-95a5-5baf0bebe1e8","podName":"fail-k8s-job-554g7-head-wt9l4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:46:01.324Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d4268561-0b4f-4a2b-95a5-5baf0bebe1e8","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"fail-k8s-job-554g7-head-svc"},"newHead":{"podName":"fail-k8s-job-554g7-head-wt9l4","serviceName":"fail-k8s-job-554g7-head-svc"}}
{"level":"info","ts":"2026-06-05T09:46:01.324Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d4268561-0b4f-4a2b-95a5-5baf0bebe1e8","name":"fail-k8s-job-554g7","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:45:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-k8s-job-554g7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:46:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-554g7-head-wt9l4","serviceName":"fail-k8s-job-554g7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:46:01.337Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"ed49c437-a1ac-4976-bb99-0b5938234a87"}
{"level":"info","ts":"2026-06-05T09:46:01.337Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"ed49c437-a1ac-4976-bb99-0b5938234a87"}
{"level":"info","ts":"2026-06-05T09:46:01.337Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"ed49c437-a1ac-4976-bb99-0b5938234a87","1 head service found":"fail-k8s-job-554g7-head-svc"}
{"level":"info","ts":"2026-06-05T09:46:01.337Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"ed49c437-a1ac-4976-bb99-0b5938234a87","Found 1 head Pod":"fail-k8s-job-554g7-head-wt9l4","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:46:01.337Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"ed49c437-a1ac-4976-bb99-0b5938234a87","head Pod":"fail-k8s-job-554g7-head-wt9l4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-554g7-head-wt9l4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:46:01.337Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"42529a3b-f9fe-42f2-bf7b-94b76938d424","cluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:46:01.337Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"ed49c437-a1ac-4976-bb99-0b5938234a87","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:46:01.338Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"ed49c437-a1ac-4976-bb99-0b5938234a87","worker Pod":"fail-k8s-job-554g7-small-group-worker-svfxq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-554g7-small-group-worker-svfxq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:46:01.338Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"ed49c437-a1ac-4976-bb99-0b5938234a87","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:46:01.338Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"ed49c437-a1ac-4976-bb99-0b5938234a87","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:46:01.338Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"ed49c437-a1ac-4976-bb99-0b5938234a87","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:46:01.338Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"a3356d96-e185-4603-a45f-73cc42a7b229","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:46:01.338Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"a3356d96-e185-4603-a45f-73cc42a7b229","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:46:01.338Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"ed49c437-a1ac-4976-bb99-0b5938234a87","podName":"fail-k8s-job-554g7-head-wt9l4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:46:01.338Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"ed49c437-a1ac-4976-bb99-0b5938234a87","podName":"fail-k8s-job-554g7-head-wt9l4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:46:01.337Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"430c4e88-876b-44db-b846-826ae22b5764","rayCluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:46:01.338Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"ed49c437-a1ac-4976-bb99-0b5938234a87","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"fail-k8s-job-554g7-head-svc"},"newHead":{"podName":"fail-k8s-job-554g7-head-wt9l4","serviceName":"fail-k8s-job-554g7-head-svc"}}
{"level":"info","ts":"2026-06-05T09:46:01.338Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"a3356d96-e185-4603-a45f-73cc42a7b229","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:46:01.338Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"ed49c437-a1ac-4976-bb99-0b5938234a87","name":"fail-k8s-job-554g7","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:45:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-k8s-job-554g7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:46:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-554g7-head-wt9l4","serviceName":"fail-k8s-job-554g7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:46:01.338Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"430c4e88-876b-44db-b846-826ae22b5764","rayCluster":"fail-k8s-job-554g7","podIPs":[]}
{"level":"info","ts":"2026-06-05T09:46:01.338Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"a3356d96-e185-4603-a45f-73cc42a7b229"}
{"level":"info","ts":"2026-06-05T09:46:01.338Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"a3356d96-e185-4603-a45f-73cc42a7b229","RayCluster":"fail-k8s-job-554g7","State":""}
{"level":"info","ts":"2026-06-05T09:46:01.405Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"ed49c437-a1ac-4976-bb99-0b5938234a87","name":"fail-k8s-job-554g7","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-554g7\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7"}}
{"level":"info","ts":"2026-06-05T09:46:01.406Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"ed49c437-a1ac-4976-bb99-0b5938234a87"}
{"level":"error","ts":"2026-06-05T09:46:01.406Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"ed49c437-a1ac-4976-bb99-0b5938234a87","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-554g7\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:46:01.406Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"8b4400e9-d8da-4164-8378-d78daf9906d2"}
{"level":"info","ts":"2026-06-05T09:46:01.406Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"8b4400e9-d8da-4164-8378-d78daf9906d2"}
{"level":"info","ts":"2026-06-05T09:46:01.406Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"8b4400e9-d8da-4164-8378-d78daf9906d2","1 head service found":"fail-k8s-job-554g7-head-svc"}
{"level":"info","ts":"2026-06-05T09:46:01.406Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"8b4400e9-d8da-4164-8378-d78daf9906d2","Found 1 head Pod":"fail-k8s-job-554g7-head-wt9l4","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:46:01.406Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"8b4400e9-d8da-4164-8378-d78daf9906d2","head Pod":"fail-k8s-job-554g7-head-wt9l4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-554g7-head-wt9l4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:46:01.406Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"8b4400e9-d8da-4164-8378-d78daf9906d2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:46:01.406Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"8b4400e9-d8da-4164-8378-d78daf9906d2","worker Pod":"fail-k8s-job-554g7-small-group-worker-svfxq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-554g7-small-group-worker-svfxq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:46:01.406Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"8b4400e9-d8da-4164-8378-d78daf9906d2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:46:01.406Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"8b4400e9-d8da-4164-8378-d78daf9906d2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:46:01.406Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"8b4400e9-d8da-4164-8378-d78daf9906d2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:46:01.406Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"8b4400e9-d8da-4164-8378-d78daf9906d2","podName":"fail-k8s-job-554g7-head-wt9l4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:46:01.406Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"8b4400e9-d8da-4164-8378-d78daf9906d2","podName":"fail-k8s-job-554g7-head-wt9l4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:46:01.406Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"8b4400e9-d8da-4164-8378-d78daf9906d2","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-05T09:46:01.406Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"8b4400e9-d8da-4164-8378-d78daf9906d2","name":"fail-k8s-job-554g7","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:46:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-554g7-head-wt9l4","serviceName":"fail-k8s-job-554g7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:46:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-554g7-head-wt9l4","serviceName":"fail-k8s-job-554g7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:46:01.411Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"42529a3b-f9fe-42f2-bf7b-94b76938d424","name":"fail-k8s-job-554g7-head"}
{"level":"info","ts":"2026-06-05T09:46:01.419Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d3ad7041-f89e-44ee-b110-97f62c59f6dd"}
{"level":"info","ts":"2026-06-05T09:46:01.419Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d3ad7041-f89e-44ee-b110-97f62c59f6dd"}
{"level":"info","ts":"2026-06-05T09:46:01.419Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d3ad7041-f89e-44ee-b110-97f62c59f6dd","1 head service found":"fail-k8s-job-554g7-head-svc"}
{"level":"info","ts":"2026-06-05T09:46:01.419Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"7fc27d37-da37-4cb0-a3ae-5da648d630d1","rayCluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:46:01.419Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d3ad7041-f89e-44ee-b110-97f62c59f6dd","Found 1 head Pod":"fail-k8s-job-554g7-head-wt9l4","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:46:01.419Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d3ad7041-f89e-44ee-b110-97f62c59f6dd","head Pod":"fail-k8s-job-554g7-head-wt9l4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-554g7-head-wt9l4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:46:01.419Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d3ad7041-f89e-44ee-b110-97f62c59f6dd","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:46:01.419Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d3ad7041-f89e-44ee-b110-97f62c59f6dd","worker Pod":"fail-k8s-job-554g7-small-group-worker-svfxq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-554g7-small-group-worker-svfxq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:46:01.419Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d3ad7041-f89e-44ee-b110-97f62c59f6dd","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:46:01.419Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d3ad7041-f89e-44ee-b110-97f62c59f6dd","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:46:01.419Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d3ad7041-f89e-44ee-b110-97f62c59f6dd","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:46:01.419Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d3ad7041-f89e-44ee-b110-97f62c59f6dd","podName":"fail-k8s-job-554g7-head-wt9l4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:46:01.419Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"74d4f8e4-6886-498e-bc55-cd4170f61eff","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:46:01.419Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"74d4f8e4-6886-498e-bc55-cd4170f61eff","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:46:01.419Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d3ad7041-f89e-44ee-b110-97f62c59f6dd","podName":"fail-k8s-job-554g7-head-wt9l4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:46:01.419Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"7fc27d37-da37-4cb0-a3ae-5da648d630d1","rayCluster":"fail-k8s-job-554g7","podIPs":[]}
{"level":"info","ts":"2026-06-05T09:46:01.419Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"74d4f8e4-6886-498e-bc55-cd4170f61eff","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:46:01.419Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d3ad7041-f89e-44ee-b110-97f62c59f6dd","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-05T09:46:01.419Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d3ad7041-f89e-44ee-b110-97f62c59f6dd","name":"fail-k8s-job-554g7","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:46:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-554g7-head-wt9l4","serviceName":"fail-k8s-job-554g7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:46:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-554g7-head-wt9l4","serviceName":"fail-k8s-job-554g7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:46:01.419Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"74d4f8e4-6886-498e-bc55-cd4170f61eff"}
{"level":"info","ts":"2026-06-05T09:46:01.419Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"74d4f8e4-6886-498e-bc55-cd4170f61eff","RayCluster":"fail-k8s-job-554g7","State":""}
{"level":"info","ts":"2026-06-05T09:46:01.424Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"42529a3b-f9fe-42f2-bf7b-94b76938d424","name":"fail-k8s-job-554g7-workers"}
{"level":"info","ts":"2026-06-05T09:46:01.424Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"42529a3b-f9fe-42f2-bf7b-94b76938d424","cluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:46:01.424Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"098666b9-d3af-4f62-bdee-26b6fa25a837","cluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:46:01.426Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d3ad7041-f89e-44ee-b110-97f62c59f6dd","name":"fail-k8s-job-554g7","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-554g7\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7"}}
{"level":"info","ts":"2026-06-05T09:46:01.426Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d3ad7041-f89e-44ee-b110-97f62c59f6dd"}
{"level":"error","ts":"2026-06-05T09:46:01.426Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"d3ad7041-f89e-44ee-b110-97f62c59f6dd","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-554g7\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:46:01.431Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"034a0505-d39c-4d3f-a9f7-077afce156db"}
{"level":"info","ts":"2026-06-05T09:46:01.431Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"034a0505-d39c-4d3f-a9f7-077afce156db"}
{"level":"info","ts":"2026-06-05T09:46:01.432Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"034a0505-d39c-4d3f-a9f7-077afce156db","1 head service found":"fail-k8s-job-554g7-head-svc"}
{"level":"info","ts":"2026-06-05T09:46:01.432Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"034a0505-d39c-4d3f-a9f7-077afce156db","Found 1 head Pod":"fail-k8s-job-554g7-head-wt9l4","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:46:01.432Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"034a0505-d39c-4d3f-a9f7-077afce156db","head Pod":"fail-k8s-job-554g7-head-wt9l4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-554g7-head-wt9l4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:46:01.432Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"034a0505-d39c-4d3f-a9f7-077afce156db","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:46:01.432Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"034a0505-d39c-4d3f-a9f7-077afce156db","worker Pod":"fail-k8s-job-554g7-small-group-worker-svfxq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-554g7-small-group-worker-svfxq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:46:01.432Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"034a0505-d39c-4d3f-a9f7-077afce156db","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:46:01.432Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"034a0505-d39c-4d3f-a9f7-077afce156db","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:46:01.432Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"034a0505-d39c-4d3f-a9f7-077afce156db","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:46:01.432Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"034a0505-d39c-4d3f-a9f7-077afce156db","podName":"fail-k8s-job-554g7-head-wt9l4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:46:01.432Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"034a0505-d39c-4d3f-a9f7-077afce156db","podName":"fail-k8s-job-554g7-head-wt9l4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:46:01.432Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"034a0505-d39c-4d3f-a9f7-077afce156db","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:46:01.432Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"034a0505-d39c-4d3f-a9f7-077afce156db","seconds":300}
{"level":"info","ts":"2026-06-05T09:46:01.438Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"098666b9-d3af-4f62-bdee-26b6fa25a837","name":"fail-k8s-job-554g7-head"}
{"level":"info","ts":"2026-06-05T09:46:01.448Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"098666b9-d3af-4f62-bdee-26b6fa25a837","name":"fail-k8s-job-554g7-workers"}
{"level":"info","ts":"2026-06-05T09:46:01.448Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"098666b9-d3af-4f62-bdee-26b6fa25a837","cluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:46:01.505Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"0b93866a-96ab-4c4f-973d-ece119abec30","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:46:01.506Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"0b93866a-96ab-4c4f-973d-ece119abec30","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:46:01.506Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"0b93866a-96ab-4c4f-973d-ece119abec30","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:46:01.506Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"0b93866a-96ab-4c4f-973d-ece119abec30"}
{"level":"info","ts":"2026-06-05T09:46:01.506Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"0b93866a-96ab-4c4f-973d-ece119abec30","RayCluster":"fail-k8s-job-554g7","State":""}
{"level":"info","ts":"2026-06-05T09:46:01.762Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"f1421f98-1503-4dfe-93e2-910884a6cb1a"}
{"level":"info","ts":"2026-06-05T09:46:01.762Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"f1421f98-1503-4dfe-93e2-910884a6cb1a"}
{"level":"info","ts":"2026-06-05T09:46:01.762Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"f1421f98-1503-4dfe-93e2-910884a6cb1a","1 head service found":"fail-k8s-job-554g7-head-svc"}
{"level":"info","ts":"2026-06-05T09:46:01.762Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"f1421f98-1503-4dfe-93e2-910884a6cb1a","Found 1 head Pod":"fail-k8s-job-554g7-head-wt9l4","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:46:01.763Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"f1421f98-1503-4dfe-93e2-910884a6cb1a","head Pod":"fail-k8s-job-554g7-head-wt9l4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-554g7-head-wt9l4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:46:01.763Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"f1421f98-1503-4dfe-93e2-910884a6cb1a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:46:01.763Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"f1421f98-1503-4dfe-93e2-910884a6cb1a","worker Pod":"fail-k8s-job-554g7-small-group-worker-svfxq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-554g7-small-group-worker-svfxq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:46:01.763Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"f1421f98-1503-4dfe-93e2-910884a6cb1a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:46:01.763Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"f1421f98-1503-4dfe-93e2-910884a6cb1a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:46:01.763Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"f1421f98-1503-4dfe-93e2-910884a6cb1a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:46:01.763Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"f1421f98-1503-4dfe-93e2-910884a6cb1a","podName":"fail-k8s-job-554g7-head-wt9l4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:46:01.763Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"f1421f98-1503-4dfe-93e2-910884a6cb1a","podName":"fail-k8s-job-554g7-head-wt9l4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:46:01.763Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"f1421f98-1503-4dfe-93e2-910884a6cb1a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:46:01.763Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"f1421f98-1503-4dfe-93e2-910884a6cb1a","seconds":300}
{"level":"info","ts":"2026-06-05T09:46:01.794Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"ca7d9024-27a5-4759-8643-8d9d000b8f4f"}
{"level":"info","ts":"2026-06-05T09:46:01.794Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"ca7d9024-27a5-4759-8643-8d9d000b8f4f"}
{"level":"info","ts":"2026-06-05T09:46:01.794Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"ca7d9024-27a5-4759-8643-8d9d000b8f4f","1 head service found":"fail-k8s-job-554g7-head-svc"}
{"level":"info","ts":"2026-06-05T09:46:01.794Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"ca7d9024-27a5-4759-8643-8d9d000b8f4f","Found 1 head Pod":"fail-k8s-job-554g7-head-wt9l4","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:46:01.794Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"ca7d9024-27a5-4759-8643-8d9d000b8f4f","head Pod":"fail-k8s-job-554g7-head-wt9l4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-554g7-head-wt9l4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:46:01.794Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"ca7d9024-27a5-4759-8643-8d9d000b8f4f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:46:01.794Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"ca7d9024-27a5-4759-8643-8d9d000b8f4f","worker Pod":"fail-k8s-job-554g7-small-group-worker-svfxq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-554g7-small-group-worker-svfxq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:46:01.794Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"ca7d9024-27a5-4759-8643-8d9d000b8f4f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:46:01.794Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"ca7d9024-27a5-4759-8643-8d9d000b8f4f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:46:01.794Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"ca7d9024-27a5-4759-8643-8d9d000b8f4f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:46:01.794Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"ca7d9024-27a5-4759-8643-8d9d000b8f4f","podName":"fail-k8s-job-554g7-head-wt9l4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:46:01.794Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"ca7d9024-27a5-4759-8643-8d9d000b8f4f","podName":"fail-k8s-job-554g7-head-wt9l4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:46:01.795Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"ca7d9024-27a5-4759-8643-8d9d000b8f4f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:46:01.795Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"ca7d9024-27a5-4759-8643-8d9d000b8f4f","seconds":300}
{"level":"info","ts":"2026-06-05T09:46:02.159Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"589a7afd-0f72-47ad-b775-569b432ceee8"}
{"level":"info","ts":"2026-06-05T09:46:02.159Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"589a7afd-0f72-47ad-b775-569b432ceee8"}
{"level":"info","ts":"2026-06-05T09:46:02.159Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"589a7afd-0f72-47ad-b775-569b432ceee8","1 head service found":"fail-k8s-job-554g7-head-svc"}
{"level":"info","ts":"2026-06-05T09:46:02.160Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"589a7afd-0f72-47ad-b775-569b432ceee8","Found 1 head Pod":"fail-k8s-job-554g7-head-wt9l4","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:46:02.166Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"589a7afd-0f72-47ad-b775-569b432ceee8","head Pod":"fail-k8s-job-554g7-head-wt9l4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-554g7-head-wt9l4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:46:02.166Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"589a7afd-0f72-47ad-b775-569b432ceee8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:46:02.166Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"589a7afd-0f72-47ad-b775-569b432ceee8","worker Pod":"fail-k8s-job-554g7-small-group-worker-svfxq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-554g7-small-group-worker-svfxq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:46:02.166Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"589a7afd-0f72-47ad-b775-569b432ceee8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:46:02.166Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"589a7afd-0f72-47ad-b775-569b432ceee8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:46:02.166Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"589a7afd-0f72-47ad-b775-569b432ceee8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:46:02.166Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"589a7afd-0f72-47ad-b775-569b432ceee8","podName":"fail-k8s-job-554g7-head-wt9l4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:46:02.166Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"589a7afd-0f72-47ad-b775-569b432ceee8","podName":"fail-k8s-job-554g7-head-wt9l4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:46:02.166Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"589a7afd-0f72-47ad-b775-569b432ceee8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:46:02.166Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"589a7afd-0f72-47ad-b775-569b432ceee8","seconds":300}
{"level":"info","ts":"2026-06-05T09:46:02.306Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"72e9ad3b-782a-40c1-bc09-0a1ad4bfaca3"}
{"level":"info","ts":"2026-06-05T09:46:02.306Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"72e9ad3b-782a-40c1-bc09-0a1ad4bfaca3"}
{"level":"info","ts":"2026-06-05T09:46:02.306Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"72e9ad3b-782a-40c1-bc09-0a1ad4bfaca3","1 head service found":"fail-k8s-job-554g7-head-svc"}
{"level":"info","ts":"2026-06-05T09:46:02.307Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"72e9ad3b-782a-40c1-bc09-0a1ad4bfaca3","Found 1 head Pod":"fail-k8s-job-554g7-head-wt9l4","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:46:02.307Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"72e9ad3b-782a-40c1-bc09-0a1ad4bfaca3","head Pod":"fail-k8s-job-554g7-head-wt9l4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-554g7-head-wt9l4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:46:02.307Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"72e9ad3b-782a-40c1-bc09-0a1ad4bfaca3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:46:02.307Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"72e9ad3b-782a-40c1-bc09-0a1ad4bfaca3","worker Pod":"fail-k8s-job-554g7-small-group-worker-svfxq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-554g7-small-group-worker-svfxq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:46:02.307Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"72e9ad3b-782a-40c1-bc09-0a1ad4bfaca3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:46:02.307Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"72e9ad3b-782a-40c1-bc09-0a1ad4bfaca3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:46:02.307Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"72e9ad3b-782a-40c1-bc09-0a1ad4bfaca3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:46:02.307Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"72e9ad3b-782a-40c1-bc09-0a1ad4bfaca3","podName":"fail-k8s-job-554g7-head-wt9l4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:46:02.307Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"72e9ad3b-782a-40c1-bc09-0a1ad4bfaca3","podName":"fail-k8s-job-554g7-head-wt9l4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:46:02.307Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"72e9ad3b-782a-40c1-bc09-0a1ad4bfaca3","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"fail-k8s-job-554g7-head-wt9l4","serviceName":"fail-k8s-job-554g7-head-svc"},"newHead":{"podIP":"10.132.0.24","serviceIP":"10.132.0.24","podName":"fail-k8s-job-554g7-head-wt9l4","serviceName":"fail-k8s-job-554g7-head-svc"}}
{"level":"info","ts":"2026-06-05T09:46:02.308Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"72e9ad3b-782a-40c1-bc09-0a1ad4bfaca3","name":"fail-k8s-job-554g7","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:46:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-554g7-head-wt9l4","serviceName":"fail-k8s-job-554g7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:46:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.24","serviceIP":"10.132.0.24","podName":"fail-k8s-job-554g7-head-wt9l4","serviceName":"fail-k8s-job-554g7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:46:02.322Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"90a5a1c1-021f-457c-b777-7b4f91ca25b6","rayCluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:46:02.322Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"91bccd96-c601-4cf3-b4b3-f77eb2dcf67f","cluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:46:02.322Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"90a5a1c1-021f-457c-b777-7b4f91ca25b6","pod":"fail-k8s-job-554g7-head-wt9l4","ip":"10.132.0.24"}
{"level":"info","ts":"2026-06-05T09:46:02.322Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"90a5a1c1-021f-457c-b777-7b4f91ca25b6","pod":"fail-k8s-job-554g7-small-group-worker-svfxq","ip":"10.133.0.47"}
{"level":"info","ts":"2026-06-05T09:46:02.322Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"f251e3f4-0cd7-4841-9152-caa835812f50","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:46:02.322Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"f251e3f4-0cd7-4841-9152-caa835812f50","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:46:02.322Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"f251e3f4-0cd7-4841-9152-caa835812f50","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:46:02.322Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"f251e3f4-0cd7-4841-9152-caa835812f50"}
{"level":"info","ts":"2026-06-05T09:46:02.322Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"f251e3f4-0cd7-4841-9152-caa835812f50","RayCluster":"fail-k8s-job-554g7","State":""}
{"level":"info","ts":"2026-06-05T09:46:02.335Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"91bccd96-c601-4cf3-b4b3-f77eb2dcf67f","name":"fail-k8s-job-554g7-head"}
{"level":"info","ts":"2026-06-05T09:46:02.335Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"90a5a1c1-021f-457c-b777-7b4f91ca25b6","certificate":"ray-head-cert-fail-k8s-job-554g7","dnsNames":["fail-k8s-job-554g7-head-svc","localhost","fail-k8s-job-554g7-head-svc.test-ns-thrrw.svc","fail-k8s-job-554g7-head-svc.test-ns-thrrw.svc.cluster.local"],"ipAddresses":["10.132.0.24","10.133.0.47","127.0.0.1"]}
{"level":"info","ts":"2026-06-05T09:46:02.345Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"91bccd96-c601-4cf3-b4b3-f77eb2dcf67f","name":"fail-k8s-job-554g7-workers"}
{"level":"info","ts":"2026-06-05T09:46:02.345Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"91bccd96-c601-4cf3-b4b3-f77eb2dcf67f","cluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:46:02.354Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"90a5a1c1-021f-457c-b777-7b4f91ca25b6","certificate":"ray-worker-cert-fail-k8s-job-554g7","dnsNames":["fail-k8s-job-554g7-worker-svc","localhost","fail-k8s-job-554g7-small-group","fail-k8s-job-554g7-worker-svc.test-ns-thrrw.svc","fail-k8s-job-554g7-worker-svc.test-ns-thrrw.svc.cluster.local","*.fail-k8s-job-554g7-worker-svc.test-ns-thrrw.svc","*.fail-k8s-job-554g7-worker-svc.test-ns-thrrw.svc.cluster.local","*-worker-*.test-ns-thrrw.svc","*-worker-*.test-ns-thrrw.svc.cluster.local"],"ipAddresses":["10.132.0.24","10.133.0.47","127.0.0.1"]}
{"level":"info","ts":"2026-06-05T09:46:02.354Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"90a5a1c1-021f-457c-b777-7b4f91ca25b6"}
{"level":"info","ts":"2026-06-05T09:46:04.322Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"b365ca9e-bf17-4204-aeef-002f1a1bd1db"}
{"level":"info","ts":"2026-06-05T09:46:04.322Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"b365ca9e-bf17-4204-aeef-002f1a1bd1db"}
{"level":"info","ts":"2026-06-05T09:46:04.322Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"b365ca9e-bf17-4204-aeef-002f1a1bd1db","1 head service found":"fail-k8s-job-554g7-head-svc"}
{"level":"info","ts":"2026-06-05T09:46:04.322Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"b365ca9e-bf17-4204-aeef-002f1a1bd1db","Found 1 head Pod":"fail-k8s-job-554g7-head-wt9l4","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:46:04.322Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"b365ca9e-bf17-4204-aeef-002f1a1bd1db","head Pod":"fail-k8s-job-554g7-head-wt9l4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-554g7-head-wt9l4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:46:04.322Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"b365ca9e-bf17-4204-aeef-002f1a1bd1db","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:46:04.322Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"b365ca9e-bf17-4204-aeef-002f1a1bd1db","worker Pod":"fail-k8s-job-554g7-small-group-worker-svfxq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-554g7-small-group-worker-svfxq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:46:04.322Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"b365ca9e-bf17-4204-aeef-002f1a1bd1db","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:46:04.322Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"b365ca9e-bf17-4204-aeef-002f1a1bd1db","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:46:04.322Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"b365ca9e-bf17-4204-aeef-002f1a1bd1db","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:46:04.322Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"b365ca9e-bf17-4204-aeef-002f1a1bd1db","podName":"fail-k8s-job-554g7-head-wt9l4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:46:04.322Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"b365ca9e-bf17-4204-aeef-002f1a1bd1db","podName":"fail-k8s-job-554g7-head-wt9l4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:46:04.322Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"b365ca9e-bf17-4204-aeef-002f1a1bd1db","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:46:04.322Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"b365ca9e-bf17-4204-aeef-002f1a1bd1db","seconds":300}
{"level":"info","ts":"2026-06-05T09:46:04.506Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"65f70a88-3659-45a1-8027-a4d6f6702191","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:46:04.506Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"65f70a88-3659-45a1-8027-a4d6f6702191","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:46:04.506Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"65f70a88-3659-45a1-8027-a4d6f6702191","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:46:04.506Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"65f70a88-3659-45a1-8027-a4d6f6702191"}
{"level":"info","ts":"2026-06-05T09:46:04.506Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"65f70a88-3659-45a1-8027-a4d6f6702191","RayCluster":"fail-k8s-job-554g7","State":""}
{"level":"info","ts":"2026-06-05T09:46:07.507Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"a6189761-497b-48f1-a510-19f65107dfcd","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:46:07.507Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"a6189761-497b-48f1-a510-19f65107dfcd","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:46:07.507Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"a6189761-497b-48f1-a510-19f65107dfcd","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:46:07.508Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"a6189761-497b-48f1-a510-19f65107dfcd"}
{"level":"info","ts":"2026-06-05T09:46:07.508Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"a6189761-497b-48f1-a510-19f65107dfcd","RayCluster":"fail-k8s-job-554g7","State":""}
{"level":"info","ts":"2026-06-05T09:46:10.509Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"82bc651e-45c3-4a6b-94e7-bd605c238969","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:46:10.509Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"82bc651e-45c3-4a6b-94e7-bd605c238969","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:46:10.509Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"82bc651e-45c3-4a6b-94e7-bd605c238969","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:46:10.509Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"82bc651e-45c3-4a6b-94e7-bd605c238969"}
{"level":"info","ts":"2026-06-05T09:46:10.509Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"82bc651e-45c3-4a6b-94e7-bd605c238969","RayCluster":"fail-k8s-job-554g7","State":""}
{"level":"info","ts":"2026-06-05T09:46:13.509Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"bc3f38fa-6d51-4387-a119-af39aa5f21aa","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:46:13.509Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"bc3f38fa-6d51-4387-a119-af39aa5f21aa","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:46:13.509Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"bc3f38fa-6d51-4387-a119-af39aa5f21aa","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:46:13.509Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"bc3f38fa-6d51-4387-a119-af39aa5f21aa"}
{"level":"info","ts":"2026-06-05T09:46:13.509Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"bc3f38fa-6d51-4387-a119-af39aa5f21aa","RayCluster":"fail-k8s-job-554g7","State":""}
{"level":"info","ts":"2026-06-05T09:46:16.510Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"139cd9f1-2cd2-4fab-9d61-5d01183e4321","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:46:16.510Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"139cd9f1-2cd2-4fab-9d61-5d01183e4321","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:46:16.510Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"139cd9f1-2cd2-4fab-9d61-5d01183e4321","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:46:16.510Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"139cd9f1-2cd2-4fab-9d61-5d01183e4321"}
{"level":"info","ts":"2026-06-05T09:46:16.510Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"139cd9f1-2cd2-4fab-9d61-5d01183e4321","RayCluster":"fail-k8s-job-554g7","State":""}
{"level":"info","ts":"2026-06-05T09:46:19.510Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"6f590c23-c459-4e30-8d93-a12260ed178d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:46:19.510Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"6f590c23-c459-4e30-8d93-a12260ed178d","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:46:19.510Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"6f590c23-c459-4e30-8d93-a12260ed178d","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:46:19.511Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"6f590c23-c459-4e30-8d93-a12260ed178d"}
{"level":"info","ts":"2026-06-05T09:46:19.511Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"6f590c23-c459-4e30-8d93-a12260ed178d","RayCluster":"fail-k8s-job-554g7","State":""}
{"level":"info","ts":"2026-06-05T09:46:22.511Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"94918818-4bf8-4171-bd2f-d9ec839200d8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:46:22.511Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"94918818-4bf8-4171-bd2f-d9ec839200d8","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:46:22.511Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"94918818-4bf8-4171-bd2f-d9ec839200d8","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:46:22.511Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"94918818-4bf8-4171-bd2f-d9ec839200d8"}
{"level":"info","ts":"2026-06-05T09:46:22.511Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"94918818-4bf8-4171-bd2f-d9ec839200d8","RayCluster":"fail-k8s-job-554g7","State":""}
{"level":"info","ts":"2026-06-05T09:46:25.512Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"9940a000-92b5-4e1e-a444-974bdc76415c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:46:25.512Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"9940a000-92b5-4e1e-a444-974bdc76415c","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:46:25.512Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"9940a000-92b5-4e1e-a444-974bdc76415c","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:46:25.512Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"9940a000-92b5-4e1e-a444-974bdc76415c"}
{"level":"info","ts":"2026-06-05T09:46:25.512Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"9940a000-92b5-4e1e-a444-974bdc76415c","RayCluster":"fail-k8s-job-554g7","State":""}
{"level":"info","ts":"2026-06-05T09:46:28.513Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"33a34bed-ff7b-4ba9-b7c9-8f632ecce9f1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:46:28.513Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"33a34bed-ff7b-4ba9-b7c9-8f632ecce9f1","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:46:28.513Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"33a34bed-ff7b-4ba9-b7c9-8f632ecce9f1","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:46:28.513Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"33a34bed-ff7b-4ba9-b7c9-8f632ecce9f1"}
{"level":"info","ts":"2026-06-05T09:46:28.513Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"33a34bed-ff7b-4ba9-b7c9-8f632ecce9f1","RayCluster":"fail-k8s-job-554g7","State":""}
{"level":"info","ts":"2026-06-05T09:46:28.720Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"72a9c820-8159-4a77-a354-8b59f0ccf040","rayCluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:46:28.720Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"72a9c820-8159-4a77-a354-8b59f0ccf040","pod":"fail-k8s-job-554g7-head-wt9l4","ip":"10.132.0.24"}
{"level":"info","ts":"2026-06-05T09:46:28.720Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"72a9c820-8159-4a77-a354-8b59f0ccf040","pod":"fail-k8s-job-554g7-small-group-worker-svfxq","ip":"10.133.0.47"}
{"level":"info","ts":"2026-06-05T09:46:28.721Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"72a9c820-8159-4a77-a354-8b59f0ccf040","rayCluster":"fail-k8s-job-554g7","podIPs":["10.132.0.24","10.133.0.47"]}
{"level":"info","ts":"2026-06-05T09:46:31.514Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"038c117d-bb4e-40f2-9d8d-757003809bb2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:46:31.514Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"038c117d-bb4e-40f2-9d8d-757003809bb2","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:46:31.514Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"038c117d-bb4e-40f2-9d8d-757003809bb2","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:46:31.514Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"038c117d-bb4e-40f2-9d8d-757003809bb2"}
{"level":"info","ts":"2026-06-05T09:46:31.514Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"038c117d-bb4e-40f2-9d8d-757003809bb2","RayCluster":"fail-k8s-job-554g7","State":""}
{"level":"info","ts":"2026-06-05T09:46:34.515Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"bdc408c4-80a9-4991-a31c-552820cbe669","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:46:34.515Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"bdc408c4-80a9-4991-a31c-552820cbe669","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:46:34.515Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"bdc408c4-80a9-4991-a31c-552820cbe669","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:46:34.515Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"bdc408c4-80a9-4991-a31c-552820cbe669"}
{"level":"info","ts":"2026-06-05T09:46:34.515Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"bdc408c4-80a9-4991-a31c-552820cbe669","RayCluster":"fail-k8s-job-554g7","State":""}
{"level":"info","ts":"2026-06-05T09:46:36.554Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"15f0c3d5-c031-4fe1-8928-c099d80714f7","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:46:36.555Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"15f0c3d5-c031-4fe1-8928-c099d80714f7","pod":"raycluster-head-vjd7d","ip":"10.134.0.20"}
{"level":"info","ts":"2026-06-05T09:46:36.555Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"15f0c3d5-c031-4fe1-8928-c099d80714f7","pod":"raycluster-small-group-worker-j9pbw","ip":"10.133.0.35"}
{"level":"info","ts":"2026-06-05T09:46:36.555Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"15f0c3d5-c031-4fe1-8928-c099d80714f7","rayCluster":"raycluster","podIPs":["10.134.0.20","10.133.0.35"]}
{"level":"info","ts":"2026-06-05T09:46:37.516Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"9d95defe-d9ac-416b-ab49-fd69ee209dd7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:46:37.516Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"9d95defe-d9ac-416b-ab49-fd69ee209dd7","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:46:37.516Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"9d95defe-d9ac-416b-ab49-fd69ee209dd7","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:46:37.516Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"9d95defe-d9ac-416b-ab49-fd69ee209dd7"}
{"level":"info","ts":"2026-06-05T09:46:37.516Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"9d95defe-d9ac-416b-ab49-fd69ee209dd7","RayCluster":"fail-k8s-job-554g7","State":""}
{"level":"info","ts":"2026-06-05T09:46:40.517Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"52438af8-b414-49de-82e9-68c5cf5694c9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:46:40.517Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"52438af8-b414-49de-82e9-68c5cf5694c9","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:46:40.517Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"52438af8-b414-49de-82e9-68c5cf5694c9","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:46:40.518Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"52438af8-b414-49de-82e9-68c5cf5694c9"}
{"level":"info","ts":"2026-06-05T09:46:40.518Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"52438af8-b414-49de-82e9-68c5cf5694c9","RayCluster":"fail-k8s-job-554g7","State":""}
{"level":"info","ts":"2026-06-05T09:46:43.518Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"6fa81878-abb4-457b-a914-059adc35c8de","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:46:43.518Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"6fa81878-abb4-457b-a914-059adc35c8de","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:46:43.518Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"6fa81878-abb4-457b-a914-059adc35c8de","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:46:43.518Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"6fa81878-abb4-457b-a914-059adc35c8de"}
{"level":"info","ts":"2026-06-05T09:46:43.518Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"6fa81878-abb4-457b-a914-059adc35c8de","RayCluster":"fail-k8s-job-554g7","State":""}
{"level":"info","ts":"2026-06-05T09:46:46.519Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"bafa86ef-426f-41c9-8011-84f789e003f7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:46:46.520Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"bafa86ef-426f-41c9-8011-84f789e003f7","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:46:46.520Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"bafa86ef-426f-41c9-8011-84f789e003f7","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:46:46.520Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"bafa86ef-426f-41c9-8011-84f789e003f7"}
{"level":"info","ts":"2026-06-05T09:46:46.520Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"bafa86ef-426f-41c9-8011-84f789e003f7","RayCluster":"fail-k8s-job-554g7","State":""}
{"level":"info","ts":"2026-06-05T09:46:49.521Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"14c1b76e-495a-4b23-9012-e2f7dd1f0f94","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:46:49.521Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"14c1b76e-495a-4b23-9012-e2f7dd1f0f94","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:46:49.521Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"14c1b76e-495a-4b23-9012-e2f7dd1f0f94","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:46:49.521Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"14c1b76e-495a-4b23-9012-e2f7dd1f0f94"}
{"level":"info","ts":"2026-06-05T09:46:49.521Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"14c1b76e-495a-4b23-9012-e2f7dd1f0f94","RayCluster":"fail-k8s-job-554g7","State":""}
{"level":"info","ts":"2026-06-05T09:46:52.521Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"51fa6d04-9dfb-41a6-a6f6-640d17fd492d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:46:52.521Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"51fa6d04-9dfb-41a6-a6f6-640d17fd492d","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:46:52.521Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"51fa6d04-9dfb-41a6-a6f6-640d17fd492d","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:46:52.521Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"51fa6d04-9dfb-41a6-a6f6-640d17fd492d"}
{"level":"info","ts":"2026-06-05T09:46:52.521Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"51fa6d04-9dfb-41a6-a6f6-640d17fd492d","RayCluster":"fail-k8s-job-554g7","State":""}
{"level":"info","ts":"2026-06-05T09:46:55.522Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"f055e1db-8dc2-4371-9e0f-7b1de6e7e0c3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:46:55.522Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"f055e1db-8dc2-4371-9e0f-7b1de6e7e0c3","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:46:55.522Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"f055e1db-8dc2-4371-9e0f-7b1de6e7e0c3","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:46:55.523Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"f055e1db-8dc2-4371-9e0f-7b1de6e7e0c3"}
{"level":"info","ts":"2026-06-05T09:46:55.523Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"f055e1db-8dc2-4371-9e0f-7b1de6e7e0c3","RayCluster":"fail-k8s-job-554g7","State":""}
{"level":"info","ts":"2026-06-05T09:46:58.524Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"f207e08d-973c-40a7-8aa5-6395468e580b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:46:58.524Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"f207e08d-973c-40a7-8aa5-6395468e580b","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:46:58.524Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"f207e08d-973c-40a7-8aa5-6395468e580b","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:46:58.524Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"f207e08d-973c-40a7-8aa5-6395468e580b"}
{"level":"info","ts":"2026-06-05T09:46:58.524Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"f207e08d-973c-40a7-8aa5-6395468e580b","RayCluster":"fail-k8s-job-554g7","State":""}
{"level":"info","ts":"2026-06-05T09:47:01.524Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"8b99512c-3e6a-44af-89fe-f53433fd6002","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:47:01.524Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"8b99512c-3e6a-44af-89fe-f53433fd6002","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:01.525Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"8b99512c-3e6a-44af-89fe-f53433fd6002","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:47:01.525Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"8b99512c-3e6a-44af-89fe-f53433fd6002"}
{"level":"info","ts":"2026-06-05T09:47:01.525Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"8b99512c-3e6a-44af-89fe-f53433fd6002","RayCluster":"fail-k8s-job-554g7","State":""}
{"level":"info","ts":"2026-06-05T09:47:04.525Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"40405670-2fb5-4747-9d34-5374453cf52f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:47:04.525Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"40405670-2fb5-4747-9d34-5374453cf52f","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:04.525Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"40405670-2fb5-4747-9d34-5374453cf52f","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:47:04.526Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"40405670-2fb5-4747-9d34-5374453cf52f"}
{"level":"info","ts":"2026-06-05T09:47:04.526Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"40405670-2fb5-4747-9d34-5374453cf52f","RayCluster":"fail-k8s-job-554g7","State":""}
{"level":"info","ts":"2026-06-05T09:47:07.527Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"4c1f6d46-6f23-4a67-ab13-81a499c78bf6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:47:07.527Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"4c1f6d46-6f23-4a67-ab13-81a499c78bf6","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:07.527Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"4c1f6d46-6f23-4a67-ab13-81a499c78bf6","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:47:07.527Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"4c1f6d46-6f23-4a67-ab13-81a499c78bf6"}
{"level":"info","ts":"2026-06-05T09:47:07.527Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"4c1f6d46-6f23-4a67-ab13-81a499c78bf6","RayCluster":"fail-k8s-job-554g7","State":""}
{"level":"info","ts":"2026-06-05T09:47:10.528Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"0193bef2-22e6-48a3-a55e-4618ae99cea6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:47:10.528Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"0193bef2-22e6-48a3-a55e-4618ae99cea6","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:10.528Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"0193bef2-22e6-48a3-a55e-4618ae99cea6","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:47:10.528Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"0193bef2-22e6-48a3-a55e-4618ae99cea6"}
{"level":"info","ts":"2026-06-05T09:47:10.528Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"0193bef2-22e6-48a3-a55e-4618ae99cea6","RayCluster":"fail-k8s-job-554g7","State":""}
{"level":"info","ts":"2026-06-05T09:47:13.529Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"6f20475f-f93a-454f-9f25-e2c1d6f7a902","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:47:13.529Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"6f20475f-f93a-454f-9f25-e2c1d6f7a902","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:13.529Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"6f20475f-f93a-454f-9f25-e2c1d6f7a902","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:47:13.529Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"6f20475f-f93a-454f-9f25-e2c1d6f7a902"}
{"level":"info","ts":"2026-06-05T09:47:13.529Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"6f20475f-f93a-454f-9f25-e2c1d6f7a902","RayCluster":"fail-k8s-job-554g7","State":""}
{"level":"info","ts":"2026-06-05T09:47:16.530Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"31a6811b-de06-4248-bb93-359c2d9e5dc2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:47:16.530Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"31a6811b-de06-4248-bb93-359c2d9e5dc2","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:16.530Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"31a6811b-de06-4248-bb93-359c2d9e5dc2","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:47:16.530Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"31a6811b-de06-4248-bb93-359c2d9e5dc2"}
{"level":"info","ts":"2026-06-05T09:47:16.530Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"31a6811b-de06-4248-bb93-359c2d9e5dc2","RayCluster":"fail-k8s-job-554g7","State":""}
{"level":"info","ts":"2026-06-05T09:47:19.531Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"9c0cc61f-bd93-48fb-9e68-d79ea8ba07a9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:47:19.531Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"9c0cc61f-bd93-48fb-9e68-d79ea8ba07a9","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:19.531Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"9c0cc61f-bd93-48fb-9e68-d79ea8ba07a9","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:47:19.531Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"9c0cc61f-bd93-48fb-9e68-d79ea8ba07a9"}
{"level":"info","ts":"2026-06-05T09:47:19.531Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"9c0cc61f-bd93-48fb-9e68-d79ea8ba07a9","RayCluster":"fail-k8s-job-554g7","State":""}
{"level":"info","ts":"2026-06-05T09:47:22.464Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"99e8ecf1-a40f-4d82-830d-d008de6e7800"}
{"level":"info","ts":"2026-06-05T09:47:22.464Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"99e8ecf1-a40f-4d82-830d-d008de6e7800"}
{"level":"info","ts":"2026-06-05T09:47:22.464Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"99e8ecf1-a40f-4d82-830d-d008de6e7800","1 head service found":"fail-k8s-job-554g7-head-svc"}
{"level":"info","ts":"2026-06-05T09:47:22.464Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"99e8ecf1-a40f-4d82-830d-d008de6e7800","Found 1 head Pod":"fail-k8s-job-554g7-head-wt9l4","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:47:22.464Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"99e8ecf1-a40f-4d82-830d-d008de6e7800","head Pod":"fail-k8s-job-554g7-head-wt9l4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-554g7-head-wt9l4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:47:22.464Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"99e8ecf1-a40f-4d82-830d-d008de6e7800","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:47:22.464Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"99e8ecf1-a40f-4d82-830d-d008de6e7800","worker Pod":"fail-k8s-job-554g7-small-group-worker-svfxq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-554g7-small-group-worker-svfxq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:47:22.464Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"99e8ecf1-a40f-4d82-830d-d008de6e7800","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:47:22.464Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"99e8ecf1-a40f-4d82-830d-d008de6e7800","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:47:22.464Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"99e8ecf1-a40f-4d82-830d-d008de6e7800","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:47:22.465Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"99e8ecf1-a40f-4d82-830d-d008de6e7800","podName":"fail-k8s-job-554g7-head-wt9l4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:47:22.465Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"99e8ecf1-a40f-4d82-830d-d008de6e7800","podName":"fail-k8s-job-554g7-head-wt9l4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:47:22.465Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"99e8ecf1-a40f-4d82-830d-d008de6e7800","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:47:22.465Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"99e8ecf1-a40f-4d82-830d-d008de6e7800","seconds":300}
{"level":"info","ts":"2026-06-05T09:47:22.532Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"8b2cbcc7-91c8-4c9c-80ac-8425af6f0ca9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:47:22.532Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"8b2cbcc7-91c8-4c9c-80ac-8425af6f0ca9","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:22.532Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"8b2cbcc7-91c8-4c9c-80ac-8425af6f0ca9","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:47:22.532Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"8b2cbcc7-91c8-4c9c-80ac-8425af6f0ca9"}
{"level":"info","ts":"2026-06-05T09:47:22.532Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"8b2cbcc7-91c8-4c9c-80ac-8425af6f0ca9","RayCluster":"fail-k8s-job-554g7","State":""}
{"level":"info","ts":"2026-06-05T09:47:23.475Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"6bd6a2f6-54ee-4c9c-a508-f4013881516d"}
{"level":"info","ts":"2026-06-05T09:47:23.475Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"6bd6a2f6-54ee-4c9c-a508-f4013881516d"}
{"level":"info","ts":"2026-06-05T09:47:23.475Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"6bd6a2f6-54ee-4c9c-a508-f4013881516d","1 head service found":"fail-k8s-job-554g7-head-svc"}
{"level":"info","ts":"2026-06-05T09:47:23.475Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"6bd6a2f6-54ee-4c9c-a508-f4013881516d","Found 1 head Pod":"fail-k8s-job-554g7-head-wt9l4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:47:23.475Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"6bd6a2f6-54ee-4c9c-a508-f4013881516d","head Pod":"fail-k8s-job-554g7-head-wt9l4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-554g7-head-wt9l4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:47:23.475Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"6bd6a2f6-54ee-4c9c-a508-f4013881516d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:47:23.475Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"6bd6a2f6-54ee-4c9c-a508-f4013881516d","worker Pod":"fail-k8s-job-554g7-small-group-worker-svfxq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-554g7-small-group-worker-svfxq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:47:23.475Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"6bd6a2f6-54ee-4c9c-a508-f4013881516d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:47:23.475Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"6bd6a2f6-54ee-4c9c-a508-f4013881516d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:47:23.475Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"6bd6a2f6-54ee-4c9c-a508-f4013881516d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:47:23.476Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"6bd6a2f6-54ee-4c9c-a508-f4013881516d","podName":"fail-k8s-job-554g7-head-wt9l4","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:46:01Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:47:23.476Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"6bd6a2f6-54ee-4c9c-a508-f4013881516d","podName":"fail-k8s-job-554g7-head-wt9l4","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:46:01Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:47:23.476Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"6bd6a2f6-54ee-4c9c-a508-f4013881516d","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-05T09:47:23.476Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"6bd6a2f6-54ee-4c9c-a508-f4013881516d","name":"fail-k8s-job-554g7","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:46:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.24","serviceIP":"10.132.0.24","podName":"fail-k8s-job-554g7-head-wt9l4","serviceName":"fail-k8s-job-554g7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:47:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.24","serviceIP":"10.132.0.24","podName":"fail-k8s-job-554g7-head-wt9l4","serviceName":"fail-k8s-job-554g7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:47:23.507Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"f5e3fcbd-d0d5-4c34-a310-6efa639245a5","cluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:23.507Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"8ce9af54-9be9-46a3-85ea-be41b9423ffa","rayCluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:23.507Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"8ce9af54-9be9-46a3-85ea-be41b9423ffa","pod":"fail-k8s-job-554g7-head-wt9l4","ip":"10.132.0.24"}
{"level":"info","ts":"2026-06-05T09:47:23.507Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"8ce9af54-9be9-46a3-85ea-be41b9423ffa","pod":"fail-k8s-job-554g7-small-group-worker-svfxq","ip":"10.133.0.47"}
{"level":"info","ts":"2026-06-05T09:47:23.508Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"8ce9af54-9be9-46a3-85ea-be41b9423ffa","rayCluster":"fail-k8s-job-554g7","podIPs":["10.132.0.24","10.133.0.47"]}
{"level":"info","ts":"2026-06-05T09:47:23.508Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"21019ab0-ea4f-4bb4-b43f-92e05e599f9d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:47:23.508Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"21019ab0-ea4f-4bb4-b43f-92e05e599f9d","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:23.508Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"21019ab0-ea4f-4bb4-b43f-92e05e599f9d","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:47:23.508Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"21019ab0-ea4f-4bb4-b43f-92e05e599f9d"}
{"level":"info","ts":"2026-06-05T09:47:23.508Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"21019ab0-ea4f-4bb4-b43f-92e05e599f9d","RayCluster":"fail-k8s-job-554g7","State":""}
{"level":"info","ts":"2026-06-05T09:47:23.528Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"f5e3fcbd-d0d5-4c34-a310-6efa639245a5","name":"fail-k8s-job-554g7-head"}
{"level":"info","ts":"2026-06-05T09:47:23.612Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"f5e3fcbd-d0d5-4c34-a310-6efa639245a5","name":"fail-k8s-job-554g7-workers"}
{"level":"info","ts":"2026-06-05T09:47:23.612Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"f5e3fcbd-d0d5-4c34-a310-6efa639245a5","cluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:25.507Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"8b6fe931-2fbf-4507-91a9-330aa6095c56"}
{"level":"info","ts":"2026-06-05T09:47:25.507Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"8b6fe931-2fbf-4507-91a9-330aa6095c56"}
{"level":"info","ts":"2026-06-05T09:47:25.507Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"8b6fe931-2fbf-4507-91a9-330aa6095c56","1 head service found":"fail-k8s-job-554g7-head-svc"}
{"level":"info","ts":"2026-06-05T09:47:25.507Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"8b6fe931-2fbf-4507-91a9-330aa6095c56","Found 1 head Pod":"fail-k8s-job-554g7-head-wt9l4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:47:25.507Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"8b6fe931-2fbf-4507-91a9-330aa6095c56","head Pod":"fail-k8s-job-554g7-head-wt9l4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-554g7-head-wt9l4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:47:25.507Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"8b6fe931-2fbf-4507-91a9-330aa6095c56","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:47:25.507Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"8b6fe931-2fbf-4507-91a9-330aa6095c56","worker Pod":"fail-k8s-job-554g7-small-group-worker-svfxq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-554g7-small-group-worker-svfxq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:47:25.507Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"8b6fe931-2fbf-4507-91a9-330aa6095c56","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:47:25.507Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"8b6fe931-2fbf-4507-91a9-330aa6095c56","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:47:25.507Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"8b6fe931-2fbf-4507-91a9-330aa6095c56","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:47:25.507Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"8b6fe931-2fbf-4507-91a9-330aa6095c56","podName":"fail-k8s-job-554g7-head-wt9l4","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:46:01Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:47:25.507Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"8b6fe931-2fbf-4507-91a9-330aa6095c56","podName":"fail-k8s-job-554g7-head-wt9l4","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:46:01Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:47:25.507Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"8b6fe931-2fbf-4507-91a9-330aa6095c56","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:47:25.507Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"8b6fe931-2fbf-4507-91a9-330aa6095c56","seconds":300}
{"level":"info","ts":"2026-06-05T09:47:25.533Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"5772cfc4-6abc-43c8-8d99-16b5147a1cbc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:47:25.533Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"5772cfc4-6abc-43c8-8d99-16b5147a1cbc","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:25.533Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"5772cfc4-6abc-43c8-8d99-16b5147a1cbc","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:47:25.533Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"5772cfc4-6abc-43c8-8d99-16b5147a1cbc"}
{"level":"info","ts":"2026-06-05T09:47:25.533Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"5772cfc4-6abc-43c8-8d99-16b5147a1cbc","RayCluster":"fail-k8s-job-554g7","State":""}
{"level":"info","ts":"2026-06-05T09:47:28.534Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"e51a9985-9cbf-4c41-9fb1-f1e01dec0eca","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:47:28.534Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"e51a9985-9cbf-4c41-9fb1-f1e01dec0eca","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:28.534Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"e51a9985-9cbf-4c41-9fb1-f1e01dec0eca","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:47:28.534Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"e51a9985-9cbf-4c41-9fb1-f1e01dec0eca"}
{"level":"info","ts":"2026-06-05T09:47:28.534Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"e51a9985-9cbf-4c41-9fb1-f1e01dec0eca","RayCluster":"fail-k8s-job-554g7","State":""}
{"level":"info","ts":"2026-06-05T09:47:28.562Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"f9e73263-3a2e-43a3-bd7a-6a04c3ff4007"}
{"level":"info","ts":"2026-06-05T09:47:28.562Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"f9e73263-3a2e-43a3-bd7a-6a04c3ff4007"}
{"level":"info","ts":"2026-06-05T09:47:28.562Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"f9e73263-3a2e-43a3-bd7a-6a04c3ff4007","1 head service found":"fail-k8s-job-554g7-head-svc"}
{"level":"info","ts":"2026-06-05T09:47:28.562Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"f9e73263-3a2e-43a3-bd7a-6a04c3ff4007","Found 1 head Pod":"fail-k8s-job-554g7-head-wt9l4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:47:28.562Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"f9e73263-3a2e-43a3-bd7a-6a04c3ff4007","head Pod":"fail-k8s-job-554g7-head-wt9l4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-554g7-head-wt9l4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:47:28.562Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"f9e73263-3a2e-43a3-bd7a-6a04c3ff4007","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:47:28.562Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"f9e73263-3a2e-43a3-bd7a-6a04c3ff4007","worker Pod":"fail-k8s-job-554g7-small-group-worker-svfxq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-554g7-small-group-worker-svfxq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:47:28.562Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"f9e73263-3a2e-43a3-bd7a-6a04c3ff4007","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:47:28.562Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"f9e73263-3a2e-43a3-bd7a-6a04c3ff4007","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:47:28.562Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"f9e73263-3a2e-43a3-bd7a-6a04c3ff4007","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:47:28.563Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"f9e73263-3a2e-43a3-bd7a-6a04c3ff4007","podName":"fail-k8s-job-554g7-head-wt9l4","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:46:01Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:47:28.563Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"f9e73263-3a2e-43a3-bd7a-6a04c3ff4007","podName":"fail-k8s-job-554g7-head-wt9l4","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:46:01Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:47:28.563Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"f9e73263-3a2e-43a3-bd7a-6a04c3ff4007","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:47:28.563Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"f9e73263-3a2e-43a3-bd7a-6a04c3ff4007","seconds":300}
{"level":"info","ts":"2026-06-05T09:47:28.721Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"6fb3e63a-1d5e-493e-a1db-cc499188a636","rayCluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:28.721Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"6fb3e63a-1d5e-493e-a1db-cc499188a636","pod":"fail-k8s-job-554g7-head-wt9l4","ip":"10.132.0.24"}
{"level":"info","ts":"2026-06-05T09:47:28.721Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"6fb3e63a-1d5e-493e-a1db-cc499188a636","pod":"fail-k8s-job-554g7-small-group-worker-svfxq","ip":"10.133.0.47"}
{"level":"info","ts":"2026-06-05T09:47:28.721Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"6fb3e63a-1d5e-493e-a1db-cc499188a636","rayCluster":"fail-k8s-job-554g7","podIPs":["10.132.0.24","10.133.0.47"]}
{"level":"info","ts":"2026-06-05T09:47:29.577Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"7fbfc025-7db9-4e83-847e-db5dc158754f"}
{"level":"info","ts":"2026-06-05T09:47:29.577Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"7fbfc025-7db9-4e83-847e-db5dc158754f"}
{"level":"info","ts":"2026-06-05T09:47:29.577Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"7fbfc025-7db9-4e83-847e-db5dc158754f","1 head service found":"fail-k8s-job-554g7-head-svc"}
{"level":"info","ts":"2026-06-05T09:47:29.577Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"7fbfc025-7db9-4e83-847e-db5dc158754f","Found 1 head Pod":"fail-k8s-job-554g7-head-wt9l4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:47:29.577Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"7fbfc025-7db9-4e83-847e-db5dc158754f","head Pod":"fail-k8s-job-554g7-head-wt9l4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-554g7-head-wt9l4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:47:29.577Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"7fbfc025-7db9-4e83-847e-db5dc158754f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:47:29.577Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"7fbfc025-7db9-4e83-847e-db5dc158754f","worker Pod":"fail-k8s-job-554g7-small-group-worker-svfxq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-554g7-small-group-worker-svfxq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:47:29.577Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"7fbfc025-7db9-4e83-847e-db5dc158754f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:47:29.577Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"7fbfc025-7db9-4e83-847e-db5dc158754f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:47:29.577Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"7fbfc025-7db9-4e83-847e-db5dc158754f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:47:29.577Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"7fbfc025-7db9-4e83-847e-db5dc158754f","podName":"fail-k8s-job-554g7-head-wt9l4","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:46:01Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:47:29.577Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"7fbfc025-7db9-4e83-847e-db5dc158754f","podName":"fail-k8s-job-554g7-head-wt9l4","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:46:01Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:47:29.577Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"7fbfc025-7db9-4e83-847e-db5dc158754f","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-05T09:47:29.577Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"7fbfc025-7db9-4e83-847e-db5dc158754f","name":"fail-k8s-job-554g7","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:47:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.24","serviceIP":"10.132.0.24","podName":"fail-k8s-job-554g7-head-wt9l4","serviceName":"fail-k8s-job-554g7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:47:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.24","serviceIP":"10.132.0.24","podName":"fail-k8s-job-554g7-head-wt9l4","serviceName":"fail-k8s-job-554g7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:47:29.590Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"aba06e0b-ad84-4930-9b70-7df77e437857","rayCluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:29.590Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"290a1569-d617-43ff-9e04-4a5ae5a6a3a0","cluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:29.590Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"4dd094dc-72ce-4c7f-a7f6-e8aafd7e6bb9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:47:29.590Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"4dd094dc-72ce-4c7f-a7f6-e8aafd7e6bb9","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:29.590Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"aba06e0b-ad84-4930-9b70-7df77e437857","pod":"fail-k8s-job-554g7-head-wt9l4","ip":"10.132.0.24"}
{"level":"info","ts":"2026-06-05T09:47:29.590Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"aba06e0b-ad84-4930-9b70-7df77e437857","pod":"fail-k8s-job-554g7-small-group-worker-svfxq","ip":"10.133.0.47"}
{"level":"info","ts":"2026-06-05T09:47:29.590Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"aba06e0b-ad84-4930-9b70-7df77e437857","rayCluster":"fail-k8s-job-554g7","podIPs":["10.132.0.24","10.133.0.47"]}
{"level":"info","ts":"2026-06-05T09:47:29.590Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"4dd094dc-72ce-4c7f-a7f6-e8aafd7e6bb9","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:47:29.591Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"4dd094dc-72ce-4c7f-a7f6-e8aafd7e6bb9"}
{"level":"info","ts":"2026-06-05T09:47:29.591Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"4dd094dc-72ce-4c7f-a7f6-e8aafd7e6bb9","RayCluster":"fail-k8s-job-554g7","State":""}
{"level":"info","ts":"2026-06-05T09:47:29.608Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"290a1569-d617-43ff-9e04-4a5ae5a6a3a0","name":"fail-k8s-job-554g7-head"}
{"level":"info","ts":"2026-06-05T09:47:29.631Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"290a1569-d617-43ff-9e04-4a5ae5a6a3a0","name":"fail-k8s-job-554g7-workers"}
{"level":"info","ts":"2026-06-05T09:47:29.631Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"290a1569-d617-43ff-9e04-4a5ae5a6a3a0","cluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:31.535Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"4d3c32ce-5258-4165-aec5-b39c1062d6bd","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:47:31.535Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"4d3c32ce-5258-4165-aec5-b39c1062d6bd","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:31.535Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"4d3c32ce-5258-4165-aec5-b39c1062d6bd","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:47:31.535Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"4d3c32ce-5258-4165-aec5-b39c1062d6bd"}
{"level":"info","ts":"2026-06-05T09:47:31.535Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"4d3c32ce-5258-4165-aec5-b39c1062d6bd","RayCluster":"fail-k8s-job-554g7","State":""}
{"level":"info","ts":"2026-06-05T09:47:31.591Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"ed3b117e-a0e6-43fc-8c34-742ee92670ee"}
{"level":"info","ts":"2026-06-05T09:47:31.591Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"ed3b117e-a0e6-43fc-8c34-742ee92670ee"}
{"level":"info","ts":"2026-06-05T09:47:31.591Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"ed3b117e-a0e6-43fc-8c34-742ee92670ee","1 head service found":"fail-k8s-job-554g7-head-svc"}
{"level":"info","ts":"2026-06-05T09:47:31.591Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"ed3b117e-a0e6-43fc-8c34-742ee92670ee","Found 1 head Pod":"fail-k8s-job-554g7-head-wt9l4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:47:31.591Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"ed3b117e-a0e6-43fc-8c34-742ee92670ee","head Pod":"fail-k8s-job-554g7-head-wt9l4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-554g7-head-wt9l4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:47:31.591Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"ed3b117e-a0e6-43fc-8c34-742ee92670ee","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:47:31.591Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"ed3b117e-a0e6-43fc-8c34-742ee92670ee","worker Pod":"fail-k8s-job-554g7-small-group-worker-svfxq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-554g7-small-group-worker-svfxq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:47:31.591Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"ed3b117e-a0e6-43fc-8c34-742ee92670ee","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:47:31.591Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"ed3b117e-a0e6-43fc-8c34-742ee92670ee","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:47:31.591Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"ed3b117e-a0e6-43fc-8c34-742ee92670ee","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:47:31.591Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"ed3b117e-a0e6-43fc-8c34-742ee92670ee","podName":"fail-k8s-job-554g7-head-wt9l4","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:46:01Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:47:31.591Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"ed3b117e-a0e6-43fc-8c34-742ee92670ee","podName":"fail-k8s-job-554g7-head-wt9l4","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:46:01Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:47:31.592Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"ed3b117e-a0e6-43fc-8c34-742ee92670ee","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:47:31.592Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"ed3b117e-a0e6-43fc-8c34-742ee92670ee","seconds":300}
{"level":"info","ts":"2026-06-05T09:47:34.506Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"955be485-958f-4190-918c-cec7cb0fb1f6"}
{"level":"info","ts":"2026-06-05T09:47:34.506Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"955be485-958f-4190-918c-cec7cb0fb1f6"}
{"level":"info","ts":"2026-06-05T09:47:34.506Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"955be485-958f-4190-918c-cec7cb0fb1f6","1 head service found":"fail-k8s-job-554g7-head-svc"}
{"level":"info","ts":"2026-06-05T09:47:34.506Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"955be485-958f-4190-918c-cec7cb0fb1f6","Found 1 head Pod":"fail-k8s-job-554g7-head-wt9l4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:47:34.506Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"955be485-958f-4190-918c-cec7cb0fb1f6","head Pod":"fail-k8s-job-554g7-head-wt9l4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-554g7-head-wt9l4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:47:34.506Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"955be485-958f-4190-918c-cec7cb0fb1f6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:47:34.506Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"955be485-958f-4190-918c-cec7cb0fb1f6","worker Pod":"fail-k8s-job-554g7-small-group-worker-svfxq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-554g7-small-group-worker-svfxq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:47:34.506Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"955be485-958f-4190-918c-cec7cb0fb1f6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:47:34.506Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"955be485-958f-4190-918c-cec7cb0fb1f6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:47:34.506Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"955be485-958f-4190-918c-cec7cb0fb1f6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:47:34.506Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"955be485-958f-4190-918c-cec7cb0fb1f6","podName":"fail-k8s-job-554g7-small-group-worker-svfxq","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:46:01Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-05T09:47:34.506Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"955be485-958f-4190-918c-cec7cb0fb1f6","podName":"fail-k8s-job-554g7-small-group-worker-svfxq","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:46:01Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-05T09:47:34.506Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"955be485-958f-4190-918c-cec7cb0fb1f6","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:47:34Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-05T09:47:34.506Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"955be485-958f-4190-918c-cec7cb0fb1f6","name":"fail-k8s-job-554g7","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:47:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.24","serviceIP":"10.132.0.24","podName":"fail-k8s-job-554g7-head-wt9l4","serviceName":"fail-k8s-job-554g7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:47:34Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.24","serviceIP":"10.132.0.24","podName":"fail-k8s-job-554g7-head-wt9l4","serviceName":"fail-k8s-job-554g7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:47:34Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:47:34.519Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"c6a120ad-b8fd-45da-b3f5-db9d193db5d6","rayCluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:34.519Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"416cb8d9-5009-4bd3-a690-16dfd49973d8","cluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:34.519Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"c6a120ad-b8fd-45da-b3f5-db9d193db5d6","pod":"fail-k8s-job-554g7-head-wt9l4","ip":"10.132.0.24"}
{"level":"info","ts":"2026-06-05T09:47:34.519Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"c6a120ad-b8fd-45da-b3f5-db9d193db5d6","pod":"fail-k8s-job-554g7-small-group-worker-svfxq","ip":"10.133.0.47"}
{"level":"info","ts":"2026-06-05T09:47:34.519Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"40c6ee77-9e11-4118-a1c2-884ba3bc128c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:47:34.519Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"c6a120ad-b8fd-45da-b3f5-db9d193db5d6","rayCluster":"fail-k8s-job-554g7","podIPs":["10.132.0.24","10.133.0.47"]}
{"level":"info","ts":"2026-06-05T09:47:34.519Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"40c6ee77-9e11-4118-a1c2-884ba3bc128c","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:34.519Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"40c6ee77-9e11-4118-a1c2-884ba3bc128c","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:47:34.519Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"40c6ee77-9e11-4118-a1c2-884ba3bc128c"}
{"level":"info","ts":"2026-06-05T09:47:34.519Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"40c6ee77-9e11-4118-a1c2-884ba3bc128c","RayCluster":"fail-k8s-job-554g7","State":""}
{"level":"info","ts":"2026-06-05T09:47:34.533Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"416cb8d9-5009-4bd3-a690-16dfd49973d8","name":"fail-k8s-job-554g7-head"}
{"level":"info","ts":"2026-06-05T09:47:34.536Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"71026590-eea1-48ce-8e6a-74202a042797","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:47:34.536Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"71026590-eea1-48ce-8e6a-74202a042797","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:34.536Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"71026590-eea1-48ce-8e6a-74202a042797","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:47:34.536Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"71026590-eea1-48ce-8e6a-74202a042797"}
{"level":"info","ts":"2026-06-05T09:47:34.536Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"71026590-eea1-48ce-8e6a-74202a042797","RayCluster":"fail-k8s-job-554g7","State":""}
{"level":"info","ts":"2026-06-05T09:47:34.544Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"416cb8d9-5009-4bd3-a690-16dfd49973d8","name":"fail-k8s-job-554g7-workers"}
{"level":"info","ts":"2026-06-05T09:47:34.544Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"416cb8d9-5009-4bd3-a690-16dfd49973d8","cluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:36.519Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"a096e9dc-e419-49e0-9b0d-413987dc284d"}
{"level":"info","ts":"2026-06-05T09:47:36.519Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"a096e9dc-e419-49e0-9b0d-413987dc284d"}
{"level":"info","ts":"2026-06-05T09:47:36.519Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"a096e9dc-e419-49e0-9b0d-413987dc284d","1 head service found":"fail-k8s-job-554g7-head-svc"}
{"level":"info","ts":"2026-06-05T09:47:36.519Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"a096e9dc-e419-49e0-9b0d-413987dc284d","Found 1 head Pod":"fail-k8s-job-554g7-head-wt9l4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:47:36.519Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"a096e9dc-e419-49e0-9b0d-413987dc284d","head Pod":"fail-k8s-job-554g7-head-wt9l4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-554g7-head-wt9l4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:47:36.519Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"a096e9dc-e419-49e0-9b0d-413987dc284d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:47:36.519Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"a096e9dc-e419-49e0-9b0d-413987dc284d","worker Pod":"fail-k8s-job-554g7-small-group-worker-svfxq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-554g7-small-group-worker-svfxq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:47:36.519Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"a096e9dc-e419-49e0-9b0d-413987dc284d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:47:36.519Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"a096e9dc-e419-49e0-9b0d-413987dc284d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:47:36.519Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"a096e9dc-e419-49e0-9b0d-413987dc284d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:47:36.519Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"a096e9dc-e419-49e0-9b0d-413987dc284d","podName":"fail-k8s-job-554g7-small-group-worker-svfxq","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:46:01Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-05T09:47:36.519Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"a096e9dc-e419-49e0-9b0d-413987dc284d","podName":"fail-k8s-job-554g7-small-group-worker-svfxq","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:46:01Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-05T09:47:36.519Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"a096e9dc-e419-49e0-9b0d-413987dc284d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:47:36.519Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"a096e9dc-e419-49e0-9b0d-413987dc284d","seconds":300}
{"level":"info","ts":"2026-06-05T09:47:36.556Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"d36cf8fb-81f8-4146-888f-9eec902b605a","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:47:36.556Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"d36cf8fb-81f8-4146-888f-9eec902b605a","pod":"raycluster-head-vjd7d","ip":"10.134.0.20"}
{"level":"info","ts":"2026-06-05T09:47:36.556Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"d36cf8fb-81f8-4146-888f-9eec902b605a","pod":"raycluster-small-group-worker-j9pbw","ip":"10.133.0.35"}
{"level":"info","ts":"2026-06-05T09:47:36.556Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"d36cf8fb-81f8-4146-888f-9eec902b605a","rayCluster":"raycluster","podIPs":["10.134.0.20","10.133.0.35"]}
{"level":"info","ts":"2026-06-05T09:47:37.536Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"953c8a6e-3a1e-42e2-b5e1-bb650e0167f7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:47:37.536Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"953c8a6e-3a1e-42e2-b5e1-bb650e0167f7","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:37.536Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"953c8a6e-3a1e-42e2-b5e1-bb650e0167f7","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:47:37.537Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"953c8a6e-3a1e-42e2-b5e1-bb650e0167f7"}
{"level":"info","ts":"2026-06-05T09:47:37.537Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"953c8a6e-3a1e-42e2-b5e1-bb650e0167f7","RayCluster":"fail-k8s-job-554g7","State":""}
{"level":"info","ts":"2026-06-05T09:47:40.538Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"702b007c-9f31-4a8c-8df2-4fe2b85e8827","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:47:40.538Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"702b007c-9f31-4a8c-8df2-4fe2b85e8827","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:40.538Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"702b007c-9f31-4a8c-8df2-4fe2b85e8827","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:47:40.538Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"702b007c-9f31-4a8c-8df2-4fe2b85e8827"}
{"level":"info","ts":"2026-06-05T09:47:40.538Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"702b007c-9f31-4a8c-8df2-4fe2b85e8827","RayCluster":"fail-k8s-job-554g7","State":""}
{"level":"info","ts":"2026-06-05T09:47:40.597Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"b0b24992-b115-465a-adc5-5d6363d254b3"}
{"level":"info","ts":"2026-06-05T09:47:40.597Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"b0b24992-b115-465a-adc5-5d6363d254b3"}
{"level":"info","ts":"2026-06-05T09:47:40.598Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"b0b24992-b115-465a-adc5-5d6363d254b3","1 head service found":"fail-k8s-job-554g7-head-svc"}
{"level":"info","ts":"2026-06-05T09:47:40.598Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"b0b24992-b115-465a-adc5-5d6363d254b3","Found 1 head Pod":"fail-k8s-job-554g7-head-wt9l4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:47:40.598Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"b0b24992-b115-465a-adc5-5d6363d254b3","head Pod":"fail-k8s-job-554g7-head-wt9l4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-554g7-head-wt9l4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:47:40.598Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"b0b24992-b115-465a-adc5-5d6363d254b3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:47:40.598Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"b0b24992-b115-465a-adc5-5d6363d254b3","worker Pod":"fail-k8s-job-554g7-small-group-worker-svfxq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-554g7-small-group-worker-svfxq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:47:40.598Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"b0b24992-b115-465a-adc5-5d6363d254b3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:47:40.598Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"b0b24992-b115-465a-adc5-5d6363d254b3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:47:40.598Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"b0b24992-b115-465a-adc5-5d6363d254b3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:47:40.598Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"b0b24992-b115-465a-adc5-5d6363d254b3","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-06-05T09:47:40.598Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"b0b24992-b115-465a-adc5-5d6363d254b3","name":"fail-k8s-job-554g7","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:47:34Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.24","serviceIP":"10.132.0.24","podName":"fail-k8s-job-554g7-head-wt9l4","serviceName":"fail-k8s-job-554g7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:47:34Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:47:40Z","stateTransitionTimes":{"ready":"2026-06-05T09:47:40Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.24","serviceIP":"10.132.0.24","podName":"fail-k8s-job-554g7-head-wt9l4","serviceName":"fail-k8s-job-554g7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:47:34Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:47:40Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:45:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:45:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:47:40.611Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"ba1d4978-1c7b-4ab0-abb8-bf76c3927821","cluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:40.611Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"98f3d04c-2b56-4c98-af61-b5a508d1c1b3","rayCluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:40.611Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"98f3d04c-2b56-4c98-af61-b5a508d1c1b3","pod":"fail-k8s-job-554g7-head-wt9l4","ip":"10.132.0.24"}
{"level":"info","ts":"2026-06-05T09:47:40.611Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"98f3d04c-2b56-4c98-af61-b5a508d1c1b3","pod":"fail-k8s-job-554g7-small-group-worker-svfxq","ip":"10.133.0.47"}
{"level":"info","ts":"2026-06-05T09:47:40.611Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"98f3d04c-2b56-4c98-af61-b5a508d1c1b3","rayCluster":"fail-k8s-job-554g7","podIPs":["10.132.0.24","10.133.0.47"]}
{"level":"info","ts":"2026-06-05T09:47:40.611Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"77ab9ba6-2402-412e-8abd-6de769473812","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:47:40.611Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"77ab9ba6-2402-412e-8abd-6de769473812","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:40.611Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"77ab9ba6-2402-412e-8abd-6de769473812","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:47:40.612Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"77ab9ba6-2402-412e-8abd-6de769473812"}
{"level":"info","ts":"2026-06-05T09:47:40.612Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"77ab9ba6-2402-412e-8abd-6de769473812","head service name":"fail-k8s-job-554g7-head-svc","namespace":"test-ns-thrrw"}
{"level":"info","ts":"2026-06-05T09:47:40.612Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"77ab9ba6-2402-412e-8abd-6de769473812","head service URL":"fail-k8s-job-554g7-head-svc.test-ns-thrrw.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-05T09:47:40.612Z","logger":"controllers.RayJob","msg":"user-provided submitter template is used; the first container is assumed to be the submitter","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"77ab9ba6-2402-412e-8abd-6de769473812"}
{"level":"info","ts":"2026-06-05T09:47:40.612Z","logger":"controllers.RayJob","msg":"User-provided command is used","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"77ab9ba6-2402-412e-8abd-6de769473812","command":["ray","job","submit","--address","http://do-not-exist:8265","--","echo 123"]}
{"level":"info","ts":"2026-06-05T09:47:40.620Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"77ab9ba6-2402-412e-8abd-6de769473812","Kubernetes Job":"fail-k8s-job"}
{"level":"info","ts":"2026-06-05T09:47:40.621Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"77ab9ba6-2402-412e-8abd-6de769473812","SubmissionMode":"K8sJobMode","RayCluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:40.621Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"77ab9ba6-2402-412e-8abd-6de769473812","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-7swsf","rayClusterName":"fail-k8s-job-554g7","jobDeploymentStatus":"Initializing","startTime":"2026-06-05T09:45:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-7swsf","rayClusterName":"fail-k8s-job-554g7","dashboardURL":"fail-k8s-job-554g7-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-05T09:45:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:47:40.621Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"77ab9ba6-2402-412e-8abd-6de769473812","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-05T09:47:40.627Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"ba1d4978-1c7b-4ab0-abb8-bf76c3927821","name":"fail-k8s-job-554g7-head"}
{"level":"info","ts":"2026-06-05T09:47:40.633Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"35a21bd2-1bd7-46ec-9501-03ef3411d518","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:47:40.633Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"35a21bd2-1bd7-46ec-9501-03ef3411d518","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:40.633Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"35a21bd2-1bd7-46ec-9501-03ef3411d518","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:47:40.633Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"35a21bd2-1bd7-46ec-9501-03ef3411d518"}
{"level":"info","ts":"2026-06-05T09:47:40.633Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"35a21bd2-1bd7-46ec-9501-03ef3411d518","head service name":"fail-k8s-job-554g7-head-svc","namespace":"test-ns-thrrw"}
{"level":"info","ts":"2026-06-05T09:47:40.633Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"35a21bd2-1bd7-46ec-9501-03ef3411d518","head service URL":"fail-k8s-job-554g7-head-svc.test-ns-thrrw.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-05T09:47:40.633Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job for RayJob already exists","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"35a21bd2-1bd7-46ec-9501-03ef3411d518","Kubernetes Job":"fail-k8s-job"}
{"level":"info","ts":"2026-06-05T09:47:40.633Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"35a21bd2-1bd7-46ec-9501-03ef3411d518","SubmissionMode":"K8sJobMode","RayCluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:40.633Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"35a21bd2-1bd7-46ec-9501-03ef3411d518","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-7swsf","rayClusterName":"fail-k8s-job-554g7","jobDeploymentStatus":"Initializing","startTime":"2026-06-05T09:45:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-7swsf","rayClusterName":"fail-k8s-job-554g7","dashboardURL":"fail-k8s-job-554g7-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-05T09:45:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:47:40.633Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"35a21bd2-1bd7-46ec-9501-03ef3411d518","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-05T09:47:40.640Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"ba1d4978-1c7b-4ab0-abb8-bf76c3927821","name":"fail-k8s-job-554g7-workers"}
{"level":"info","ts":"2026-06-05T09:47:40.640Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"ba1d4978-1c7b-4ab0-abb8-bf76c3927821","cluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:40.641Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"35a21bd2-1bd7-46ec-9501-03ef3411d518","error":"Operation cannot be fulfilled on rayjobs.ray.io \"fail-k8s-job\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-06-05T09:47:40.641Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"35a21bd2-1bd7-46ec-9501-03ef3411d518"}
{"level":"error","ts":"2026-06-05T09:47:40.641Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"35a21bd2-1bd7-46ec-9501-03ef3411d518","error":"Operation cannot be fulfilled on rayjobs.ray.io \"fail-k8s-job\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:47:40.641Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"fdea783d-90e4-42b3-8905-fea986027169","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:47:40.641Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"fdea783d-90e4-42b3-8905-fea986027169","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:40.641Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"fdea783d-90e4-42b3-8905-fea986027169","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:47:40.641Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"fdea783d-90e4-42b3-8905-fea986027169"}
{"level":"error","ts":"2026-06-05T09:47:40.651Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"fdea783d-90e4-42b3-8905-fea986027169","JobId":"fail-k8s-job-7swsf","error":"Job fail-k8s-job-7swsf does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:47:40.651Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"fdea783d-90e4-42b3-8905-fea986027169"}
{"level":"error","ts":"2026-06-05T09:47:40.651Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"fdea783d-90e4-42b3-8905-fea986027169","error":"Job fail-k8s-job-7swsf does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:47:40.651Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"2934ec2d-7a3b-448f-8dc3-61c928c9e0d9","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:47:40.651Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"2934ec2d-7a3b-448f-8dc3-61c928c9e0d9","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:40.651Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"2934ec2d-7a3b-448f-8dc3-61c928c9e0d9","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:47:40.651Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"2934ec2d-7a3b-448f-8dc3-61c928c9e0d9"}
{"level":"error","ts":"2026-06-05T09:47:40.710Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"2934ec2d-7a3b-448f-8dc3-61c928c9e0d9","JobId":"fail-k8s-job-7swsf","error":"Job fail-k8s-job-7swsf does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:47:40.710Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"2934ec2d-7a3b-448f-8dc3-61c928c9e0d9"}
{"level":"error","ts":"2026-06-05T09:47:40.710Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"2934ec2d-7a3b-448f-8dc3-61c928c9e0d9","error":"Job fail-k8s-job-7swsf does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:47:40.710Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"6334716c-296d-44c6-bc32-a7e1730924e5","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:47:40.710Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"6334716c-296d-44c6-bc32-a7e1730924e5","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:40.710Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"6334716c-296d-44c6-bc32-a7e1730924e5","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:47:40.711Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"6334716c-296d-44c6-bc32-a7e1730924e5"}
{"level":"error","ts":"2026-06-05T09:47:40.715Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"6334716c-296d-44c6-bc32-a7e1730924e5","JobId":"fail-k8s-job-7swsf","error":"Job fail-k8s-job-7swsf does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:47:40.715Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"6334716c-296d-44c6-bc32-a7e1730924e5"}
{"level":"error","ts":"2026-06-05T09:47:40.715Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"6334716c-296d-44c6-bc32-a7e1730924e5","error":"Job fail-k8s-job-7swsf does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:47:40.731Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"bf50f11d-6f98-4a00-92c2-994f945aa117","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:47:40.731Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"bf50f11d-6f98-4a00-92c2-994f945aa117","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:40.731Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"bf50f11d-6f98-4a00-92c2-994f945aa117","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:47:40.732Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"bf50f11d-6f98-4a00-92c2-994f945aa117"}
{"level":"error","ts":"2026-06-05T09:47:40.736Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"bf50f11d-6f98-4a00-92c2-994f945aa117","JobId":"fail-k8s-job-7swsf","error":"Job fail-k8s-job-7swsf does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:47:40.737Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"bf50f11d-6f98-4a00-92c2-994f945aa117"}
{"level":"error","ts":"2026-06-05T09:47:40.737Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"bf50f11d-6f98-4a00-92c2-994f945aa117","error":"Job fail-k8s-job-7swsf does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:47:40.817Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"68ace3e8-5217-4656-b498-555e81394bcc","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:47:40.817Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"68ace3e8-5217-4656-b498-555e81394bcc","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:40.817Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"68ace3e8-5217-4656-b498-555e81394bcc","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:47:40.817Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"68ace3e8-5217-4656-b498-555e81394bcc"}
{"level":"error","ts":"2026-06-05T09:47:40.822Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"68ace3e8-5217-4656-b498-555e81394bcc","JobId":"fail-k8s-job-7swsf","error":"Job fail-k8s-job-7swsf does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:47:40.822Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"68ace3e8-5217-4656-b498-555e81394bcc"}
{"level":"error","ts":"2026-06-05T09:47:40.822Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"68ace3e8-5217-4656-b498-555e81394bcc","error":"Job fail-k8s-job-7swsf does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:47:40.983Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"a40e3cf3-8750-4dd8-8287-5beaa5a1d07e","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:47:40.983Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"a40e3cf3-8750-4dd8-8287-5beaa5a1d07e","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:40.983Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"a40e3cf3-8750-4dd8-8287-5beaa5a1d07e","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:47:40.983Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"a40e3cf3-8750-4dd8-8287-5beaa5a1d07e"}
{"level":"error","ts":"2026-06-05T09:47:40.989Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"a40e3cf3-8750-4dd8-8287-5beaa5a1d07e","JobId":"fail-k8s-job-7swsf","error":"Job fail-k8s-job-7swsf does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:47:40.989Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"a40e3cf3-8750-4dd8-8287-5beaa5a1d07e"}
{"level":"error","ts":"2026-06-05T09:47:40.989Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"a40e3cf3-8750-4dd8-8287-5beaa5a1d07e","error":"Job fail-k8s-job-7swsf does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:47:41.309Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"ad6242f4-fad5-4b50-b994-d587de3de9b3","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:47:41.309Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"ad6242f4-fad5-4b50-b994-d587de3de9b3","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:41.309Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"ad6242f4-fad5-4b50-b994-d587de3de9b3","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:47:41.310Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"ad6242f4-fad5-4b50-b994-d587de3de9b3"}
{"level":"error","ts":"2026-06-05T09:47:41.315Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"ad6242f4-fad5-4b50-b994-d587de3de9b3","JobId":"fail-k8s-job-7swsf","error":"Job fail-k8s-job-7swsf does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:47:41.315Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"ad6242f4-fad5-4b50-b994-d587de3de9b3"}
{"level":"error","ts":"2026-06-05T09:47:41.315Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"ad6242f4-fad5-4b50-b994-d587de3de9b3","error":"Job fail-k8s-job-7swsf does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:47:41.650Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"8ea7e14b-39a6-4e37-a846-19d71868233e","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:47:41.650Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"8ea7e14b-39a6-4e37-a846-19d71868233e","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:41.650Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"8ea7e14b-39a6-4e37-a846-19d71868233e","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:47:41.650Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"8ea7e14b-39a6-4e37-a846-19d71868233e"}
{"level":"error","ts":"2026-06-05T09:47:41.655Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"8ea7e14b-39a6-4e37-a846-19d71868233e","JobId":"fail-k8s-job-7swsf","error":"Job fail-k8s-job-7swsf does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:47:41.655Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"8ea7e14b-39a6-4e37-a846-19d71868233e"}
{"level":"error","ts":"2026-06-05T09:47:41.655Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"8ea7e14b-39a6-4e37-a846-19d71868233e","error":"Job fail-k8s-job-7swsf does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:47:41.955Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"aebab54a-faec-4d7f-85d9-55b8e6c01ded","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:47:41.956Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"aebab54a-faec-4d7f-85d9-55b8e6c01ded","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:41.956Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"aebab54a-faec-4d7f-85d9-55b8e6c01ded","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:47:41.956Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"aebab54a-faec-4d7f-85d9-55b8e6c01ded"}
{"level":"error","ts":"2026-06-05T09:47:41.961Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"aebab54a-faec-4d7f-85d9-55b8e6c01ded","JobId":"fail-k8s-job-7swsf","error":"Job fail-k8s-job-7swsf does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:47:41.961Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"aebab54a-faec-4d7f-85d9-55b8e6c01ded"}
{"level":"error","ts":"2026-06-05T09:47:41.961Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"aebab54a-faec-4d7f-85d9-55b8e6c01ded","error":"Job fail-k8s-job-7swsf does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:47:42.612Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"76c089af-0994-405f-b27e-86cb56e9846d"}
{"level":"info","ts":"2026-06-05T09:47:42.612Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"76c089af-0994-405f-b27e-86cb56e9846d"}
{"level":"info","ts":"2026-06-05T09:47:42.612Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"76c089af-0994-405f-b27e-86cb56e9846d","1 head service found":"fail-k8s-job-554g7-head-svc"}
{"level":"info","ts":"2026-06-05T09:47:42.612Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"76c089af-0994-405f-b27e-86cb56e9846d","Found 1 head Pod":"fail-k8s-job-554g7-head-wt9l4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:47:42.612Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"76c089af-0994-405f-b27e-86cb56e9846d","head Pod":"fail-k8s-job-554g7-head-wt9l4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-554g7-head-wt9l4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:47:42.612Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"76c089af-0994-405f-b27e-86cb56e9846d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:47:42.612Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"76c089af-0994-405f-b27e-86cb56e9846d","worker Pod":"fail-k8s-job-554g7-small-group-worker-svfxq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-554g7-small-group-worker-svfxq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:47:42.612Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"76c089af-0994-405f-b27e-86cb56e9846d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:47:42.612Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"76c089af-0994-405f-b27e-86cb56e9846d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:47:42.612Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"76c089af-0994-405f-b27e-86cb56e9846d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:47:42.612Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"76c089af-0994-405f-b27e-86cb56e9846d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:47:42.612Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"76c089af-0994-405f-b27e-86cb56e9846d","seconds":300}
{"level":"info","ts":"2026-06-05T09:47:44.522Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"6b5236fa-05db-41ef-9d1c-ca02ed2f9b13","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:47:44.522Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"6b5236fa-05db-41ef-9d1c-ca02ed2f9b13","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:44.522Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"6b5236fa-05db-41ef-9d1c-ca02ed2f9b13","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:47:44.522Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"6b5236fa-05db-41ef-9d1c-ca02ed2f9b13"}
{"level":"error","ts":"2026-06-05T09:47:44.527Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"6b5236fa-05db-41ef-9d1c-ca02ed2f9b13","JobId":"fail-k8s-job-7swsf","error":"Job fail-k8s-job-7swsf does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:47:44.528Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"6b5236fa-05db-41ef-9d1c-ca02ed2f9b13"}
{"level":"error","ts":"2026-06-05T09:47:44.528Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"6b5236fa-05db-41ef-9d1c-ca02ed2f9b13","error":"Job fail-k8s-job-7swsf does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:47:46.656Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"d47e334b-540e-4915-b403-f0117734aee3","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:47:46.656Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"d47e334b-540e-4915-b403-f0117734aee3","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:46.656Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"d47e334b-540e-4915-b403-f0117734aee3","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:47:46.656Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"d47e334b-540e-4915-b403-f0117734aee3"}
{"level":"error","ts":"2026-06-05T09:47:46.662Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"d47e334b-540e-4915-b403-f0117734aee3","JobId":"fail-k8s-job-7swsf","error":"Job fail-k8s-job-7swsf does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:47:46.662Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"d47e334b-540e-4915-b403-f0117734aee3"}
{"level":"error","ts":"2026-06-05T09:47:46.662Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"d47e334b-540e-4915-b403-f0117734aee3","error":"Job fail-k8s-job-7swsf does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:47:47.662Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"59748642-7b65-4bf8-a34a-8da54fbeff84","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:47:47.663Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"59748642-7b65-4bf8-a34a-8da54fbeff84","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:47.663Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"59748642-7b65-4bf8-a34a-8da54fbeff84","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:47:47.663Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"59748642-7b65-4bf8-a34a-8da54fbeff84"}
{"level":"error","ts":"2026-06-05T09:47:47.668Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"59748642-7b65-4bf8-a34a-8da54fbeff84","JobId":"fail-k8s-job-7swsf","error":"Job fail-k8s-job-7swsf does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:47:47.668Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"59748642-7b65-4bf8-a34a-8da54fbeff84"}
{"level":"error","ts":"2026-06-05T09:47:47.668Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"59748642-7b65-4bf8-a34a-8da54fbeff84","error":"Job fail-k8s-job-7swsf does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:47:47.681Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"928ae874-017f-4355-9188-9a11a0702389","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:47:47.681Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"928ae874-017f-4355-9188-9a11a0702389","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:47.681Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"928ae874-017f-4355-9188-9a11a0702389","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:47:47.681Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"928ae874-017f-4355-9188-9a11a0702389"}
{"level":"error","ts":"2026-06-05T09:47:47.686Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"928ae874-017f-4355-9188-9a11a0702389","JobId":"fail-k8s-job-7swsf","error":"Job fail-k8s-job-7swsf does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:47:47.686Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"928ae874-017f-4355-9188-9a11a0702389"}
{"level":"error","ts":"2026-06-05T09:47:47.686Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"928ae874-017f-4355-9188-9a11a0702389","error":"Job fail-k8s-job-7swsf does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:47:49.648Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"cb1e67cf-b988-4ff4-8acb-f7c26ac06468","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:47:49.648Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"cb1e67cf-b988-4ff4-8acb-f7c26ac06468","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:49.648Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"cb1e67cf-b988-4ff4-8acb-f7c26ac06468","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:47:49.649Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"cb1e67cf-b988-4ff4-8acb-f7c26ac06468"}
{"level":"error","ts":"2026-06-05T09:47:49.654Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"cb1e67cf-b988-4ff4-8acb-f7c26ac06468","JobId":"fail-k8s-job-7swsf","error":"Job fail-k8s-job-7swsf does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:47:49.654Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"cb1e67cf-b988-4ff4-8acb-f7c26ac06468"}
{"level":"error","ts":"2026-06-05T09:47:49.654Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"cb1e67cf-b988-4ff4-8acb-f7c26ac06468","error":"Job fail-k8s-job-7swsf does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:47:54.020Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"227e93f6-82e4-4521-b14e-a65da2ab7492","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:47:54.020Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"227e93f6-82e4-4521-b14e-a65da2ab7492","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:54.020Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"227e93f6-82e4-4521-b14e-a65da2ab7492","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:47:54.020Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"227e93f6-82e4-4521-b14e-a65da2ab7492"}
{"level":"error","ts":"2026-06-05T09:47:54.026Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"227e93f6-82e4-4521-b14e-a65da2ab7492","JobId":"fail-k8s-job-7swsf","error":"Job fail-k8s-job-7swsf does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:47:54.026Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"227e93f6-82e4-4521-b14e-a65da2ab7492"}
{"level":"error","ts":"2026-06-05T09:47:54.026Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"227e93f6-82e4-4521-b14e-a65da2ab7492","error":"Job fail-k8s-job-7swsf does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:47:55.019Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"f6fc48da-6dc7-4be0-92db-8d0ca6ae54ac","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:47:55.019Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"f6fc48da-6dc7-4be0-92db-8d0ca6ae54ac","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:55.019Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"f6fc48da-6dc7-4be0-92db-8d0ca6ae54ac","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:47:55.019Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"f6fc48da-6dc7-4be0-92db-8d0ca6ae54ac"}
{"level":"error","ts":"2026-06-05T09:47:55.025Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"f6fc48da-6dc7-4be0-92db-8d0ca6ae54ac","JobId":"fail-k8s-job-7swsf","error":"Job fail-k8s-job-7swsf does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:47:55.025Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"f6fc48da-6dc7-4be0-92db-8d0ca6ae54ac"}
{"level":"error","ts":"2026-06-05T09:47:55.025Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"f6fc48da-6dc7-4be0-92db-8d0ca6ae54ac","error":"Job fail-k8s-job-7swsf does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:47:59.730Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"4f2633b5-a4fd-4aec-a49b-d2a1cb7c11e3","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:47:59.730Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"4f2633b5-a4fd-4aec-a49b-d2a1cb7c11e3","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:47:59.730Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"4f2633b5-a4fd-4aec-a49b-d2a1cb7c11e3","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:47:59.730Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"4f2633b5-a4fd-4aec-a49b-d2a1cb7c11e3"}
{"level":"error","ts":"2026-06-05T09:47:59.736Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"4f2633b5-a4fd-4aec-a49b-d2a1cb7c11e3","JobId":"fail-k8s-job-7swsf","error":"Job fail-k8s-job-7swsf does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:47:59.736Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"4f2633b5-a4fd-4aec-a49b-d2a1cb7c11e3"}
{"level":"error","ts":"2026-06-05T09:47:59.736Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"4f2633b5-a4fd-4aec-a49b-d2a1cb7c11e3","error":"Job fail-k8s-job-7swsf does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:00.738Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"993db8a7-f2e6-46ef-9964-296d8f3d81ad","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:48:00.738Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"993db8a7-f2e6-46ef-9964-296d8f3d81ad","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:48:00.738Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"993db8a7-f2e6-46ef-9964-296d8f3d81ad","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:48:00.738Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"993db8a7-f2e6-46ef-9964-296d8f3d81ad"}
{"level":"error","ts":"2026-06-05T09:48:00.746Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"993db8a7-f2e6-46ef-9964-296d8f3d81ad","JobId":"fail-k8s-job-7swsf","error":"Job fail-k8s-job-7swsf does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:00.746Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"993db8a7-f2e6-46ef-9964-296d8f3d81ad"}
{"level":"error","ts":"2026-06-05T09:48:00.746Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"993db8a7-f2e6-46ef-9964-296d8f3d81ad","error":"Job fail-k8s-job-7swsf does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:00.756Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"db0413e6-561f-4559-b04b-416260187047","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:48:00.756Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"db0413e6-561f-4559-b04b-416260187047","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:48:00.756Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"db0413e6-561f-4559-b04b-416260187047","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:48:00.756Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"db0413e6-561f-4559-b04b-416260187047"}
{"level":"error","ts":"2026-06-05T09:48:00.760Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"db0413e6-561f-4559-b04b-416260187047","JobId":"fail-k8s-job-7swsf","error":"Job fail-k8s-job-7swsf does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:00.760Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"db0413e6-561f-4559-b04b-416260187047"}
{"level":"error","ts":"2026-06-05T09:48:00.760Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"db0413e6-561f-4559-b04b-416260187047","error":"Job fail-k8s-job-7swsf does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:18.024Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"bde99667-fd44-4f78-9f8b-338e8918a575","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:48:18.024Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"bde99667-fd44-4f78-9f8b-338e8918a575","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:48:18.024Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"bde99667-fd44-4f78-9f8b-338e8918a575","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:48:18.024Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"bde99667-fd44-4f78-9f8b-338e8918a575"}
{"level":"error","ts":"2026-06-05T09:48:18.032Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"bde99667-fd44-4f78-9f8b-338e8918a575","JobId":"fail-k8s-job-7swsf","error":"Job fail-k8s-job-7swsf does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:18.032Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"bde99667-fd44-4f78-9f8b-338e8918a575"}
{"level":"error","ts":"2026-06-05T09:48:18.032Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"bde99667-fd44-4f78-9f8b-338e8918a575","error":"Job fail-k8s-job-7swsf does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:19.023Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"d7bb1520-a4b4-4ffc-8b00-75dc71847f34","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:48:19.023Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"d7bb1520-a4b4-4ffc-8b00-75dc71847f34","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:48:19.024Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"d7bb1520-a4b4-4ffc-8b00-75dc71847f34","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:48:19.024Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"d7bb1520-a4b4-4ffc-8b00-75dc71847f34"}
{"level":"error","ts":"2026-06-05T09:48:19.029Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"d7bb1520-a4b4-4ffc-8b00-75dc71847f34","JobId":"fail-k8s-job-7swsf","error":"Job fail-k8s-job-7swsf does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:19.029Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"d7bb1520-a4b4-4ffc-8b00-75dc71847f34"}
{"level":"error","ts":"2026-06-05T09:48:19.029Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"d7bb1520-a4b4-4ffc-8b00-75dc71847f34","error":"Job fail-k8s-job-7swsf does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:23.847Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"a627b16a-aba0-4c14-98c0-f3891b20a535","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:48:23.847Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"a627b16a-aba0-4c14-98c0-f3891b20a535","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:48:23.847Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"a627b16a-aba0-4c14-98c0-f3891b20a535","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:48:23.847Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"a627b16a-aba0-4c14-98c0-f3891b20a535"}
{"level":"error","ts":"2026-06-05T09:48:23.853Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"a627b16a-aba0-4c14-98c0-f3891b20a535","JobId":"fail-k8s-job-7swsf","error":"Job fail-k8s-job-7swsf does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:23.853Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"a627b16a-aba0-4c14-98c0-f3891b20a535"}
{"level":"error","ts":"2026-06-05T09:48:23.853Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"a627b16a-aba0-4c14-98c0-f3891b20a535","error":"Job fail-k8s-job-7swsf does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:24.855Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"9e25fd97-2d1f-4471-9327-508379a52d51","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:48:24.855Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"9e25fd97-2d1f-4471-9327-508379a52d51","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:48:24.855Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"9e25fd97-2d1f-4471-9327-508379a52d51","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:48:24.855Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"9e25fd97-2d1f-4471-9327-508379a52d51"}
{"level":"error","ts":"2026-06-05T09:48:24.861Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"9e25fd97-2d1f-4471-9327-508379a52d51","JobId":"fail-k8s-job-7swsf","error":"Job fail-k8s-job-7swsf does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:24.861Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"9e25fd97-2d1f-4471-9327-508379a52d51"}
{"level":"error","ts":"2026-06-05T09:48:24.861Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"9e25fd97-2d1f-4471-9327-508379a52d51","error":"Job fail-k8s-job-7swsf does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:24.873Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"455e0689-7f88-412f-82b8-4e7963a504a0","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:48:24.873Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job has failed. Attempting to transition the status to `Failed`.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"455e0689-7f88-412f-82b8-4e7963a504a0","Submitter K8s Job":"fail-k8s-job","Reason":"BackoffLimitExceeded","Message":"Job has reached the specified backoff limit"}
{"level":"info","ts":"2026-06-05T09:48:24.873Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"455e0689-7f88-412f-82b8-4e7963a504a0","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-7swsf","rayClusterName":"fail-k8s-job-554g7","dashboardURL":"fail-k8s-job-554g7-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-05T09:45:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-7swsf","rayClusterName":"fail-k8s-job-554g7","dashboardURL":"fail-k8s-job-554g7-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobDeploymentStatus":"Failed","reason":"SubmissionFailed","message":"Job submission has failed. Reason: BackoffLimitExceeded. Message: Job has reached the specified backoff limit","startTime":"2026-06-05T09:45:58Z","succeeded":0,"failed":1,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:48:24.873Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"455e0689-7f88-412f-82b8-4e7963a504a0","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Failed"}
{"level":"info","ts":"2026-06-05T09:48:24.885Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"4024eb70-ac01-4c35-803b-97317a382955","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:48:24.885Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"4024eb70-ac01-4c35-803b-97317a382955","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-05 09:48:24 +0000 UTC","Now":"2026-06-05T09:48:24.885Z","ShutdownTime":"2026-06-05T09:48:24.000Z"}
{"level":"info","ts":"2026-06-05T09:48:24.885Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"4024eb70-ac01-4c35-803b-97317a382955","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-05T09:48:24.894Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob is deleted","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"4024eb70-ac01-4c35-803b-97317a382955","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:48:24.894Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"4024eb70-ac01-4c35-803b-97317a382955","isClusterDeleted":false}
{"level":"info","ts":"2026-06-05T09:48:24.894Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"4024eb70-ac01-4c35-803b-97317a382955","RayCluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:48:24.895Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"4477e640-8cf0-482f-99a0-dd61ecf0710b","rayCluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:48:24.895Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"089767d3-4f81-43af-8c8f-c7ffc8672402"}
{"level":"info","ts":"2026-06-05T09:48:24.895Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"reconcileID":"b359f9ad-5f34-427b-82b8-af88e442dcfc"}
{"level":"info","ts":"2026-06-05T09:48:24.895Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"bcdc50f9-18d5-4a76-9259-5b9ff2162906","namespacedName":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:48:24.895Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"c7ba4f2e-a159-46e5-aa86-a17f44f16daa","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:48:24.895Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"4477e640-8cf0-482f-99a0-dd61ecf0710b","pod":"fail-k8s-job-554g7-head-wt9l4","ip":"10.132.0.24"}
{"level":"info","ts":"2026-06-05T09:48:24.895Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"c7ba4f2e-a159-46e5-aa86-a17f44f16daa","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-05 09:48:24 +0000 UTC","Now":"2026-06-05T09:48:24.895Z","ShutdownTime":"2026-06-05T09:48:24.000Z"}
{"level":"info","ts":"2026-06-05T09:48:24.895Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"bcdc50f9-18d5-4a76-9259-5b9ff2162906","cluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:48:24.895Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"c7ba4f2e-a159-46e5-aa86-a17f44f16daa","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-05T09:48:24.895Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"bcdc50f9-18d5-4a76-9259-5b9ff2162906","cluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:48:24.895Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"c7ba4f2e-a159-46e5-aa86-a17f44f16daa","RayCluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:48:24.895Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"c7ba4f2e-a159-46e5-aa86-a17f44f16daa","isClusterDeleted":false}
{"level":"info","ts":"2026-06-05T09:48:24.895Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"c7ba4f2e-a159-46e5-aa86-a17f44f16daa","RayCluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:48:24.895Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"4477e640-8cf0-482f-99a0-dd61ecf0710b","pod":"fail-k8s-job-554g7-small-group-worker-svfxq","ip":"10.133.0.47"}
{"level":"info","ts":"2026-06-05T09:48:24.895Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"4477e640-8cf0-482f-99a0-dd61ecf0710b","rayCluster":"fail-k8s-job-554g7","podIPs":["10.132.0.24","10.133.0.47"]}
{"level":"info","ts":"2026-06-05T09:48:24.895Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"bcdc50f9-18d5-4a76-9259-5b9ff2162906","namespace":"test-ns-thrrw","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-05T09:48:24.901Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"bcdc50f9-18d5-4a76-9259-5b9ff2162906","name":"kuberay-gateway-access","namespace":"test-ns-thrrw"}
{"level":"info","ts":"2026-06-05T09:48:24.906Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"bcdc50f9-18d5-4a76-9259-5b9ff2162906","name":"test-ns-thrrw-fail-k8s-job-554g7","namespace":"opendatahub"}
{"level":"info","ts":"2026-06-05T09:48:24.911Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"bcdc50f9-18d5-4a76-9259-5b9ff2162906","configMap":"kube-rbac-proxy-config-fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:48:24.916Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"bcdc50f9-18d5-4a76-9259-5b9ff2162906","serviceAccount":"fail-k8s-job-554g7-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:48:24.938Z","logger":"raycluster-resource","msg":"default","name":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:48:24.938Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}
{"level":"info","ts":"2026-06-05T09:48:24.949Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"bcdc50f9-18d5-4a76-9259-5b9ff2162906","cluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:48:24.949Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"d4b88a5a-1cd5-4c12-885b-2d5490f3ccfc","namespacedName":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:48:24.949Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"d4b88a5a-1cd5-4c12-885b-2d5490f3ccfc","cluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:48:24.949Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"d4b88a5a-1cd5-4c12-885b-2d5490f3ccfc","cluster":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:48:24.949Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"d4b88a5a-1cd5-4c12-885b-2d5490f3ccfc","namespace":"test-ns-thrrw","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-05T09:48:24.949Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"d4b88a5a-1cd5-4c12-885b-2d5490f3ccfc","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-06-05T09:48:24.949Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"cb9bb243-5b4a-40e8-a674-0202ba36f1e1","namespace":"test-ns-thrrw","clusterName":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:48:24.949Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"9a2ad9eb-2488-40e7-ae72-4995e1203dad"}
{"level":"info","ts":"2026-06-05T09:48:24.949Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"1a72f0a4-0056-4377-8015-e89f9fcd344d","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:48:24.949Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"1a72f0a4-0056-4377-8015-e89f9fcd344d","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-05 09:48:24 +0000 UTC","Now":"2026-06-05T09:48:24.949Z","ShutdownTime":"2026-06-05T09:48:24.000Z"}
{"level":"info","ts":"2026-06-05T09:48:24.949Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"1a72f0a4-0056-4377-8015-e89f9fcd344d","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-05T09:48:24.949Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"1a72f0a4-0056-4377-8015-e89f9fcd344d","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:48:24.949Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"1a72f0a4-0056-4377-8015-e89f9fcd344d","isClusterDeleted":true}
{"level":"info","ts":"2026-06-05T09:48:24.949Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"1a72f0a4-0056-4377-8015-e89f9fcd344d","RayCluster":"fail-k8s-job-554g7"}
{"level":"error","ts":"2026-06-05T09:48:24.953Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"d4b88a5a-1cd5-4c12-885b-2d5490f3ccfc","error":"rayclusters.ray.io \"fail-k8s-job-554g7\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:214\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:155\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:24.953Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"d4b88a5a-1cd5-4c12-885b-2d5490f3ccfc"}
{"level":"error","ts":"2026-06-05T09:48:24.953Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"d4b88a5a-1cd5-4c12-885b-2d5490f3ccfc","error":"rayclusters.ray.io \"fail-k8s-job-554g7\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:24.953Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"a8251a55-4599-404b-bfe7-2f6bcdfea98e","namespacedName":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:48:24.953Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"a8251a55-4599-404b-bfe7-2f6bcdfea98e"}
{"level":"info","ts":"2026-06-05T09:48:24.953Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"a8251a55-4599-404b-bfe7-2f6bcdfea98e","namespace":"test-ns-thrrw","authClusters":0}
{"level":"info","ts":"2026-06-05T09:48:24.953Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"a8251a55-4599-404b-bfe7-2f6bcdfea98e","namespace":"test-ns-thrrw"}
{"level":"info","ts":"2026-06-05T09:48:24.958Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"139ff812-f306-47f3-b461-da4947ddde8c","namespacedName":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:48:24.958Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"139ff812-f306-47f3-b461-da4947ddde8c"}
{"level":"info","ts":"2026-06-05T09:48:24.958Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"139ff812-f306-47f3-b461-da4947ddde8c","namespace":"test-ns-thrrw","authClusters":0}
{"level":"info","ts":"2026-06-05T09:48:24.958Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"139ff812-f306-47f3-b461-da4947ddde8c","namespace":"test-ns-thrrw"}
{"level":"error","ts":"2026-06-05T09:48:24.960Z","msg":"Failed to delete SelfSigned issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"cb9bb243-5b4a-40e8-a674-0202ba36f1e1","issuer":"ray-selfsigned-issuer-fail-k8s-job-554g7","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-fail-k8s-job-554g7\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:254\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:25.005Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"2c1b757d-5279-4df2-9907-5fe4e37ed4b9"}
{"level":"error","ts":"2026-06-05T09:48:25.006Z","msg":"Failed to delete head certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"cb9bb243-5b4a-40e8-a674-0202ba36f1e1","certificate":"ray-head-cert-fail-k8s-job-554g7","error":"certificates.cert-manager.io \"ray-head-cert-fail-k8s-job-554g7\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:261\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:25.006Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"6afa4086-eb21-4005-b209-885d9c8cca6e","namespacedName":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:48:25.006Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"6afa4086-eb21-4005-b209-885d9c8cca6e"}
{"level":"info","ts":"2026-06-05T09:48:25.006Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"6afa4086-eb21-4005-b209-885d9c8cca6e","namespace":"test-ns-thrrw","authClusters":0}
{"level":"info","ts":"2026-06-05T09:48:25.006Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"6afa4086-eb21-4005-b209-885d9c8cca6e","namespace":"test-ns-thrrw"}
{"level":"info","ts":"2026-06-05T09:48:25.006Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"903c793c-a7ab-4214-b04e-5f34d3a23403"}
{"level":"error","ts":"2026-06-05T09:48:25.009Z","msg":"Failed to delete worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"cb9bb243-5b4a-40e8-a674-0202ba36f1e1","certificate":"ray-worker-cert-fail-k8s-job-554g7","error":"certificates.cert-manager.io \"ray-worker-cert-fail-k8s-job-554g7\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:268\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:25.026Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"cb9bb243-5b4a-40e8-a674-0202ba36f1e1","errorCount":3}
{"level":"info","ts":"2026-06-05T09:48:25.026Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"cb9bb243-5b4a-40e8-a674-0202ba36f1e1"}
{"level":"error","ts":"2026-06-05T09:48:25.026Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"cb9bb243-5b4a-40e8-a674-0202ba36f1e1","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-fail-k8s-job-554g7\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:25.031Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"122f770b-9e47-42e3-8e8a-f7317cbb7056","namespace":"test-ns-thrrw","clusterName":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:48:25.031Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-554g7","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"fail-k8s-job-554g7","reconcileID":"122f770b-9e47-42e3-8e8a-f7317cbb7056","namespace":"test-ns-thrrw","clusterName":"fail-k8s-job-554g7"}
{"level":"info","ts":"2026-06-05T09:48:25.305Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"61fd7932-1964-47f2-b152-9ebaca8c27eb","DeletionTimestamp":"2026-06-05 09:48:25 +0000 UTC"}
{"level":"error","ts":"2026-06-05T09:48:25.305Z","logger":"controllers.RayJob","msg":"Failed to get RayCluster","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"61fd7932-1964-47f2-b152-9ebaca8c27eb","error":"RayCluster.ray.io \"fail-k8s-job-554g7\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:115\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:25.305Z","logger":"controllers.RayJob","msg":"Stop a ray job","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"61fd7932-1964-47f2-b152-9ebaca8c27eb","rayJob":"fail-k8s-job-7swsf"}
{"level":"error","ts":"2026-06-05T09:48:25.308Z","logger":"controllers.RayJob","msg":"Failed to stop job for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"61fd7932-1964-47f2-b152-9ebaca8c27eb","error":"Post \"http://fail-k8s-job-554g7-head-svc.test-ns-thrrw.svc.cluster.local:8265/api/jobs/fail-k8s-job-7swsf/stop\": dial tcp 10.132.0.24:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:123\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:25.309Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"61fd7932-1964-47f2-b152-9ebaca8c27eb","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-05T09:48:25.321Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"f9c6e06c-0ac4-49e7-852e-9eea0ecc0fea","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:48:25.321Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"f9c6e06c-0ac4-49e7-852e-9eea0ecc0fea","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-05T09:48:25.332Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"f9c6e06c-0ac4-49e7-852e-9eea0ecc0fea"}
{"level":"info","ts":"2026-06-05T09:48:25.332Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"f9c6e06c-0ac4-49e7-852e-9eea0ecc0fea","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-05T09:48:25.332Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"f9c6e06c-0ac4-49e7-852e-9eea0ecc0fea","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-rnf5t","rayClusterName":"stop-z5q6j","jobDeploymentStatus":"Initializing","startTime":"2026-06-05T09:48:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:48:25.332Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"f9c6e06c-0ac4-49e7-852e-9eea0ecc0fea","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-05T09:48:25.346Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"453201b5-fbd1-4d39-8727-160e1094007a"}
{"level":"info","ts":"2026-06-05T09:48:25.346Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"9d2a1a72-eb26-4c07-957f-3f53e9a75ac3","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:48:25.346Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"9d2a1a72-eb26-4c07-957f-3f53e9a75ac3"}
{"level":"info","ts":"2026-06-05T09:48:25.346Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"9d2a1a72-eb26-4c07-957f-3f53e9a75ac3","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-05T09:48:25.346Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"9d2a1a72-eb26-4c07-957f-3f53e9a75ac3","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-c27vk","rayClusterName":"stop-dzcx7","jobDeploymentStatus":"Initializing","startTime":"2026-06-05T09:48:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:48:25.346Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"9d2a1a72-eb26-4c07-957f-3f53e9a75ac3","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-05T09:48:25.358Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"9d2a1a72-eb26-4c07-957f-3f53e9a75ac3","error":"Operation cannot be fulfilled on rayjobs.ray.io \"stop\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-06-05T09:48:25.358Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"9d2a1a72-eb26-4c07-957f-3f53e9a75ac3"}
{"level":"error","ts":"2026-06-05T09:48:25.358Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"9d2a1a72-eb26-4c07-957f-3f53e9a75ac3","error":"Operation cannot be fulfilled on rayjobs.ray.io \"stop\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:25.358Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"6430a876-dc4f-4f67-87c8-e4aba005633c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:48:25.358Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"6430a876-dc4f-4f67-87c8-e4aba005633c","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:25.358Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"6430a876-dc4f-4f67-87c8-e4aba005633c","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:48:25.358Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"6430a876-dc4f-4f67-87c8-e4aba005633c","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:48:25.362Z","logger":"raycluster-resource","msg":"default","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:25.362Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"stop-z5q6j","namespace":"test-ns-thrrw"}
{"level":"info","ts":"2026-06-05T09:48:25.380Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"6430a876-dc4f-4f67-87c8-e4aba005633c","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:48:25.380Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"b99bdb6b-e911-4b60-ba96-60b2b108795a","rayCluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:25.380Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"b99bdb6b-e911-4b60-ba96-60b2b108795a","rayCluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:25.380Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"6430a876-dc4f-4f67-87c8-e4aba005633c"}
{"level":"info","ts":"2026-06-05T09:48:25.380Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"6430a876-dc4f-4f67-87c8-e4aba005633c","RayCluster":"stop-z5q6j","State":""}
{"level":"info","ts":"2026-06-05T09:48:25.380Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"43d6ec5f-a325-4b70-9ae7-fdac6c3bcffb"}
{"level":"info","ts":"2026-06-05T09:48:25.380Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"43d6ec5f-a325-4b70-9ae7-fdac6c3bcffb"}
{"level":"info","ts":"2026-06-05T09:48:25.381Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"a2f7d67e-17ce-40dc-acfc-79f5b39b54be","namespacedName":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:48:25.381Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"a2f7d67e-17ce-40dc-acfc-79f5b39b54be","mode":"IntegratedOAuth","cluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:25.381Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"a2f7d67e-17ce-40dc-acfc-79f5b39b54be","cluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:25.381Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"a2f7d67e-17ce-40dc-acfc-79f5b39b54be","cluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:25.381Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"19cf1a38-1a8d-4b0f-97ea-d6fac57988a5","cluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:25.406Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"75be4df3-6db9-4625-80ef-2608d63638a6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:48:25.406Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"75be4df3-6db9-4625-80ef-2608d63638a6","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:25.406Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"75be4df3-6db9-4625-80ef-2608d63638a6","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:48:25.406Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"75be4df3-6db9-4625-80ef-2608d63638a6"}
{"level":"info","ts":"2026-06-05T09:48:25.406Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"75be4df3-6db9-4625-80ef-2608d63638a6","RayCluster":"stop-z5q6j","State":""}
{"level":"info","ts":"2026-06-05T09:48:25.416Z","logger":"raycluster-resource","msg":"default","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:25.416Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"stop-z5q6j","namespace":"test-ns-thrrw"}
{"level":"info","ts":"2026-06-05T09:48:25.420Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"43d6ec5f-a325-4b70-9ae7-fdac6c3bcffb","name":"stop-z5q6j-head-svc"}
{"level":"info","ts":"2026-06-05T09:48:25.420Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"43d6ec5f-a325-4b70-9ae7-fdac6c3bcffb","error":"head certificate secret ray-head-secret-stop-z5q6j not found"}
{"level":"error","ts":"2026-06-05T09:48:25.420Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"43d6ec5f-a325-4b70-9ae7-fdac6c3bcffb","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-z5q6j not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:25.420Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"43d6ec5f-a325-4b70-9ae7-fdac6c3bcffb","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-z5q6j not found"}
{"level":"info","ts":"2026-06-05T09:48:25.420Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"43d6ec5f-a325-4b70-9ae7-fdac6c3bcffb","filter labels":["test-ns-thrrw",{"ray.io/cluster":"stop-z5q6j","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:48:25.420Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"43d6ec5f-a325-4b70-9ae7-fdac6c3bcffb","filter labels":["test-ns-thrrw",{"ray.io/cluster":"stop-z5q6j","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:48:25.420Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"43d6ec5f-a325-4b70-9ae7-fdac6c3bcffb","filter labels":["test-ns-thrrw",{"ray.io/cluster":"stop-z5q6j","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:48:25.420Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"43d6ec5f-a325-4b70-9ae7-fdac6c3bcffb","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-05T09:48:25.420Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"43d6ec5f-a325-4b70-9ae7-fdac6c3bcffb","name":"stop-z5q6j","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:48:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-z5q6j-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:48:25.506Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"19cf1a38-1a8d-4b0f-97ea-d6fac57988a5","cluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:25.506Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"634fff09-9726-4c95-a7f5-2b86a364b144","cluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:25.506Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"b99bdb6b-e911-4b60-ba96-60b2b108795a","rayCluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:25.506Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"b99bdb6b-e911-4b60-ba96-60b2b108795a","rayCluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:25.506Z","logger":"raycluster-resource","msg":"default","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:25.506Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"stop-z5q6j","namespace":"test-ns-thrrw"}
{"level":"info","ts":"2026-06-05T09:48:25.506Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"43d6ec5f-a325-4b70-9ae7-fdac6c3bcffb"}
{"level":"error","ts":"2026-06-05T09:48:25.506Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"43d6ec5f-a325-4b70-9ae7-fdac6c3bcffb","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-z5q6j not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:25.507Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"2ad015dd-9b0f-4cb2-8977-b64b234bb3d6"}
{"level":"info","ts":"2026-06-05T09:48:25.507Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"2ad015dd-9b0f-4cb2-8977-b64b234bb3d6"}
{"level":"info","ts":"2026-06-05T09:48:25.507Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"2ad015dd-9b0f-4cb2-8977-b64b234bb3d6","1 head service found":"stop-z5q6j-head-svc"}
{"level":"info","ts":"2026-06-05T09:48:25.507Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"2ad015dd-9b0f-4cb2-8977-b64b234bb3d6","error":"head certificate secret ray-head-secret-stop-z5q6j not found"}
{"level":"error","ts":"2026-06-05T09:48:25.507Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"2ad015dd-9b0f-4cb2-8977-b64b234bb3d6","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-z5q6j not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:25.507Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"2ad015dd-9b0f-4cb2-8977-b64b234bb3d6","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-z5q6j not found"}
{"level":"info","ts":"2026-06-05T09:48:25.507Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"998b9cd6-e737-4edf-b74c-766db47ed689","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:48:25.507Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"998b9cd6-e737-4edf-b74c-766db47ed689","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:25.507Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"998b9cd6-e737-4edf-b74c-766db47ed689","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:48:25.507Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"2ad015dd-9b0f-4cb2-8977-b64b234bb3d6","filter labels":["test-ns-thrrw",{"ray.io/cluster":"stop-z5q6j","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:48:25.507Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"2ad015dd-9b0f-4cb2-8977-b64b234bb3d6","filter labels":["test-ns-thrrw",{"ray.io/cluster":"stop-z5q6j","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:48:25.507Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"2ad015dd-9b0f-4cb2-8977-b64b234bb3d6","filter labels":["test-ns-thrrw",{"ray.io/cluster":"stop-z5q6j","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:48:25.507Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"2ad015dd-9b0f-4cb2-8977-b64b234bb3d6"}
{"level":"error","ts":"2026-06-05T09:48:25.507Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"2ad015dd-9b0f-4cb2-8977-b64b234bb3d6","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-z5q6j not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:25.507Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"998b9cd6-e737-4edf-b74c-766db47ed689"}
{"level":"info","ts":"2026-06-05T09:48:25.507Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"998b9cd6-e737-4edf-b74c-766db47ed689","RayCluster":"stop-z5q6j","State":""}
{"level":"info","ts":"2026-06-05T09:48:25.512Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"ac15f632-9263-401b-aa4c-e934ebc38266"}
{"level":"info","ts":"2026-06-05T09:48:25.512Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"ac15f632-9263-401b-aa4c-e934ebc38266"}
{"level":"info","ts":"2026-06-05T09:48:25.512Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"ac15f632-9263-401b-aa4c-e934ebc38266","1 head service found":"stop-z5q6j-head-svc"}
{"level":"info","ts":"2026-06-05T09:48:25.512Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"ac15f632-9263-401b-aa4c-e934ebc38266","error":"head certificate secret ray-head-secret-stop-z5q6j not found"}
{"level":"error","ts":"2026-06-05T09:48:25.512Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"ac15f632-9263-401b-aa4c-e934ebc38266","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-z5q6j not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:25.512Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"ac15f632-9263-401b-aa4c-e934ebc38266","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-z5q6j not found"}
{"level":"info","ts":"2026-06-05T09:48:25.512Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"ac15f632-9263-401b-aa4c-e934ebc38266","filter labels":["test-ns-thrrw",{"ray.io/cluster":"stop-z5q6j","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:48:25.512Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"ac15f632-9263-401b-aa4c-e934ebc38266","filter labels":["test-ns-thrrw",{"ray.io/cluster":"stop-z5q6j","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:48:25.512Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"ac15f632-9263-401b-aa4c-e934ebc38266","filter labels":["test-ns-thrrw",{"ray.io/cluster":"stop-z5q6j","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:48:25.512Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"ac15f632-9263-401b-aa4c-e934ebc38266"}
{"level":"error","ts":"2026-06-05T09:48:25.512Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"ac15f632-9263-401b-aa4c-e934ebc38266","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-z5q6j not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-05T09:48:25.514Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"a2f7d67e-17ce-40dc-acfc-79f5b39b54be","error":"failed to add finalizer: Operation cannot be fulfilled on rayclusters.ray.io \"stop-z5q6j\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:25.514Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"a2f7d67e-17ce-40dc-acfc-79f5b39b54be"}
{"level":"error","ts":"2026-06-05T09:48:25.514Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"a2f7d67e-17ce-40dc-acfc-79f5b39b54be","error":"failed to add finalizer: Operation cannot be fulfilled on rayclusters.ray.io \"stop-z5q6j\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:25.514Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"ce7a4362-d476-436a-b509-e0e63ef4e1b9","namespacedName":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:48:25.514Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"ce7a4362-d476-436a-b509-e0e63ef4e1b9","mode":"IntegratedOAuth","cluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:25.514Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"ce7a4362-d476-436a-b509-e0e63ef4e1b9","cluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:25.514Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"ce7a4362-d476-436a-b509-e0e63ef4e1b9","cluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:25.528Z","logger":"raycluster-resource","msg":"default","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:25.528Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"stop-z5q6j","namespace":"test-ns-thrrw"}
{"level":"info","ts":"2026-06-05T09:48:25.535Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"b99bdb6b-e911-4b60-ba96-60b2b108795a"}
{"level":"info","ts":"2026-06-05T09:48:25.535Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"b99bdb6b-e911-4b60-ba96-60b2b108795a","rayCluster":"stop-z5q6j","secret":"ca-secret"}
{"level":"info","ts":"2026-06-05T09:48:25.535Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"b99bdb6b-e911-4b60-ba96-60b2b108795a","rayCluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:25.536Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"634fff09-9726-4c95-a7f5-2b86a364b144","name":"stop-z5q6j-head"}
{"level":"info","ts":"2026-06-05T09:48:25.549Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"f54b41ea-e246-4cf0-ad5e-812fee4c0a6e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:48:25.549Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"f54b41ea-e246-4cf0-ad5e-812fee4c0a6e","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:25.549Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"f54b41ea-e246-4cf0-ad5e-812fee4c0a6e","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:48:25.549Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"f54b41ea-e246-4cf0-ad5e-812fee4c0a6e"}
{"level":"info","ts":"2026-06-05T09:48:25.549Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"f54b41ea-e246-4cf0-ad5e-812fee4c0a6e","RayCluster":"stop-z5q6j","State":""}
{"level":"info","ts":"2026-06-05T09:48:25.552Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"b99bdb6b-e911-4b60-ba96-60b2b108795a","rayCluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:25.606Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"cdb3e7a6-b588-425e-8e3e-d0b9b9d9ef0d"}
{"level":"info","ts":"2026-06-05T09:48:25.606Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"cdb3e7a6-b588-425e-8e3e-d0b9b9d9ef0d"}
{"level":"info","ts":"2026-06-05T09:48:25.606Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"cdb3e7a6-b588-425e-8e3e-d0b9b9d9ef0d","1 head service found":"stop-z5q6j-head-svc"}
{"level":"info","ts":"2026-06-05T09:48:25.606Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"cdb3e7a6-b588-425e-8e3e-d0b9b9d9ef0d","error":"head certificate secret ray-head-secret-stop-z5q6j not found"}
{"level":"error","ts":"2026-06-05T09:48:25.606Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"cdb3e7a6-b588-425e-8e3e-d0b9b9d9ef0d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-z5q6j not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:25.606Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"cdb3e7a6-b588-425e-8e3e-d0b9b9d9ef0d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-z5q6j not found"}
{"level":"info","ts":"2026-06-05T09:48:25.606Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"b99bdb6b-e911-4b60-ba96-60b2b108795a"}
{"level":"info","ts":"2026-06-05T09:48:25.606Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"cdb3e7a6-b588-425e-8e3e-d0b9b9d9ef0d","filter labels":["test-ns-thrrw",{"ray.io/cluster":"stop-z5q6j","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:48:25.606Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"cdb3e7a6-b588-425e-8e3e-d0b9b9d9ef0d","filter labels":["test-ns-thrrw",{"ray.io/cluster":"stop-z5q6j","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:48:25.606Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"b99bdb6b-e911-4b60-ba96-60b2b108795a","rayCluster":"stop-z5q6j","podIPs":[]}
{"level":"info","ts":"2026-06-05T09:48:25.606Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"ce7a4362-d476-436a-b509-e0e63ef4e1b9","name":"stop-z5q6j-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:48:25.606Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"cdb3e7a6-b588-425e-8e3e-d0b9b9d9ef0d","filter labels":["test-ns-thrrw",{"ray.io/cluster":"stop-z5q6j","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:48:25.606Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"cdb3e7a6-b588-425e-8e3e-d0b9b9d9ef0d"}
{"level":"error","ts":"2026-06-05T09:48:25.606Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"cdb3e7a6-b588-425e-8e3e-d0b9b9d9ef0d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-z5q6j not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:25.613Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"634fff09-9726-4c95-a7f5-2b86a364b144","name":"stop-z5q6j-workers"}
{"level":"info","ts":"2026-06-05T09:48:25.613Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"634fff09-9726-4c95-a7f5-2b86a364b144","cluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:25.613Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"ee47c724-25fa-433d-8408-6053c01b35ab","cluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:25.615Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"ce7a4362-d476-436a-b509-e0e63ef4e1b9","name":"kuberay-gateway-access","namespace":"test-ns-thrrw","operation":"created"}
{"level":"info","ts":"2026-06-05T09:48:25.621Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"b99bdb6b-e911-4b60-ba96-60b2b108795a"}
{"level":"info","ts":"2026-06-05T09:48:25.621Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"b99bdb6b-e911-4b60-ba96-60b2b108795a","rayCluster":"stop-z5q6j","podIPs":[]}
{"level":"info","ts":"2026-06-05T09:48:25.621Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"b99bdb6b-e911-4b60-ba96-60b2b108795a"}
{"level":"info","ts":"2026-06-05T09:48:25.622Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"a9a2c9fa-59d3-46e7-8e1f-c0f2ff587cb3","rayCluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:25.622Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"a9a2c9fa-59d3-46e7-8e1f-c0f2ff587cb3"}
{"level":"info","ts":"2026-06-05T09:48:25.626Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"ce7a4362-d476-436a-b509-e0e63ef4e1b9","name":"test-ns-thrrw-stop-z5q6j","operation":"created"}
{"level":"info","ts":"2026-06-05T09:48:25.627Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"ee47c724-25fa-433d-8408-6053c01b35ab","name":"stop-z5q6j-head"}
{"level":"info","ts":"2026-06-05T09:48:25.634Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"ce7a4362-d476-436a-b509-e0e63ef4e1b9","name":"kube-rbac-proxy-config-stop-z5q6j","operation":"created"}
{"level":"info","ts":"2026-06-05T09:48:25.705Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"784b6aee-e5a6-4b6b-b4a2-4f1f7803bee9"}
{"level":"info","ts":"2026-06-05T09:48:25.705Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"784b6aee-e5a6-4b6b-b4a2-4f1f7803bee9"}
{"level":"info","ts":"2026-06-05T09:48:25.706Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"784b6aee-e5a6-4b6b-b4a2-4f1f7803bee9","1 head service found":"stop-z5q6j-head-svc"}
{"level":"info","ts":"2026-06-05T09:48:25.706Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"784b6aee-e5a6-4b6b-b4a2-4f1f7803bee9","error":"head certificate secret ray-head-secret-stop-z5q6j not found"}
{"level":"error","ts":"2026-06-05T09:48:25.706Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"784b6aee-e5a6-4b6b-b4a2-4f1f7803bee9","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-z5q6j not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:25.706Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"784b6aee-e5a6-4b6b-b4a2-4f1f7803bee9","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-z5q6j not found"}
{"level":"error","ts":"2026-06-05T09:48:25.706Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"ce7a4362-d476-436a-b509-e0e63ef4e1b9","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"stop-z5q6j\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:25.706Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"784b6aee-e5a6-4b6b-b4a2-4f1f7803bee9","filter labels":["test-ns-thrrw",{"ray.io/cluster":"stop-z5q6j","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:48:25.706Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"ce7a4362-d476-436a-b509-e0e63ef4e1b9"}
{"level":"info","ts":"2026-06-05T09:48:25.706Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"784b6aee-e5a6-4b6b-b4a2-4f1f7803bee9","filter labels":["test-ns-thrrw",{"ray.io/cluster":"stop-z5q6j","ray.io/node-type":"head"}]}
{"level":"error","ts":"2026-06-05T09:48:25.706Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"ce7a4362-d476-436a-b509-e0e63ef4e1b9","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"stop-z5q6j\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:25.706Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"784b6aee-e5a6-4b6b-b4a2-4f1f7803bee9","filter labels":["test-ns-thrrw",{"ray.io/cluster":"stop-z5q6j","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:48:25.706Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"ee47c724-25fa-433d-8408-6053c01b35ab","name":"stop-z5q6j-workers"}
{"level":"info","ts":"2026-06-05T09:48:25.706Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"fee1fe61-4898-4666-8f2c-b78da5ba7037","namespacedName":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:48:25.706Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"ee47c724-25fa-433d-8408-6053c01b35ab","cluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:25.706Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"784b6aee-e5a6-4b6b-b4a2-4f1f7803bee9"}
{"level":"error","ts":"2026-06-05T09:48:25.706Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"784b6aee-e5a6-4b6b-b4a2-4f1f7803bee9","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-z5q6j not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:25.706Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"fee1fe61-4898-4666-8f2c-b78da5ba7037","mode":"IntegratedOAuth","cluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:25.706Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"fee1fe61-4898-4666-8f2c-b78da5ba7037","cluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:25.719Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"fee1fe61-4898-4666-8f2c-b78da5ba7037","cluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:25.719Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"296148db-b333-4cea-b8ff-ce9bfa93f23f","namespacedName":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:48:25.719Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"296148db-b333-4cea-b8ff-ce9bfa93f23f","mode":"IntegratedOAuth","cluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:25.719Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"296148db-b333-4cea-b8ff-ce9bfa93f23f","cluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:25.719Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"cbc8927a-9269-4d43-87f4-5f788b0949f8","rayCluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:25.720Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"21a9daec-c8ac-4b39-8485-d7b96c4cff24","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:48:25.720Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"21a9daec-c8ac-4b39-8485-d7b96c4cff24","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:25.720Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"cbc8927a-9269-4d43-87f4-5f788b0949f8"}
{"level":"info","ts":"2026-06-05T09:48:25.720Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"21a9daec-c8ac-4b39-8485-d7b96c4cff24","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:48:25.720Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"21a9daec-c8ac-4b39-8485-d7b96c4cff24"}
{"level":"info","ts":"2026-06-05T09:48:25.720Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"21a9daec-c8ac-4b39-8485-d7b96c4cff24","RayCluster":"stop-z5q6j","State":""}
{"level":"info","ts":"2026-06-05T09:48:25.731Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"296148db-b333-4cea-b8ff-ce9bfa93f23f","cluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:25.805Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"12cc667b-f959-43ed-9b93-4fdb97ff0299","cluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:25.805Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"72258b71-95b1-4f78-8278-d47cf9311de1"}
{"level":"info","ts":"2026-06-05T09:48:25.806Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"72258b71-95b1-4f78-8278-d47cf9311de1"}
{"level":"info","ts":"2026-06-05T09:48:25.806Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"72258b71-95b1-4f78-8278-d47cf9311de1","1 head service found":"stop-z5q6j-head-svc"}
{"level":"info","ts":"2026-06-05T09:48:25.806Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"72258b71-95b1-4f78-8278-d47cf9311de1","error":"head certificate secret ray-head-secret-stop-z5q6j not found"}
{"level":"error","ts":"2026-06-05T09:48:25.806Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"72258b71-95b1-4f78-8278-d47cf9311de1","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-z5q6j not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:25.806Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"72258b71-95b1-4f78-8278-d47cf9311de1","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-z5q6j not found"}
{"level":"info","ts":"2026-06-05T09:48:25.806Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"72258b71-95b1-4f78-8278-d47cf9311de1","filter labels":["test-ns-thrrw",{"ray.io/cluster":"stop-z5q6j","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:48:25.806Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"72258b71-95b1-4f78-8278-d47cf9311de1","filter labels":["test-ns-thrrw",{"ray.io/cluster":"stop-z5q6j","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:48:25.806Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"72258b71-95b1-4f78-8278-d47cf9311de1","filter labels":["test-ns-thrrw",{"ray.io/cluster":"stop-z5q6j","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:48:25.806Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"72258b71-95b1-4f78-8278-d47cf9311de1"}
{"level":"error","ts":"2026-06-05T09:48:25.806Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"72258b71-95b1-4f78-8278-d47cf9311de1","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-z5q6j not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:25.819Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"12cc667b-f959-43ed-9b93-4fdb97ff0299","name":"stop-z5q6j-head"}
{"level":"info","ts":"2026-06-05T09:48:25.831Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"12cc667b-f959-43ed-9b93-4fdb97ff0299","name":"stop-z5q6j-workers"}
{"level":"info","ts":"2026-06-05T09:48:25.831Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"12cc667b-f959-43ed-9b93-4fdb97ff0299","cluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:25.967Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"186f1bc4-a6fb-4a11-a4f2-1c5dfe8637da"}
{"level":"info","ts":"2026-06-05T09:48:25.967Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"186f1bc4-a6fb-4a11-a4f2-1c5dfe8637da"}
{"level":"info","ts":"2026-06-05T09:48:25.967Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"186f1bc4-a6fb-4a11-a4f2-1c5dfe8637da","1 head service found":"stop-z5q6j-head-svc"}
{"level":"info","ts":"2026-06-05T09:48:25.967Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"186f1bc4-a6fb-4a11-a4f2-1c5dfe8637da","error":"head certificate secret ray-head-secret-stop-z5q6j not found"}
{"level":"error","ts":"2026-06-05T09:48:25.967Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"186f1bc4-a6fb-4a11-a4f2-1c5dfe8637da","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-z5q6j not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:25.967Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"186f1bc4-a6fb-4a11-a4f2-1c5dfe8637da","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-z5q6j not found"}
{"level":"info","ts":"2026-06-05T09:48:25.967Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"186f1bc4-a6fb-4a11-a4f2-1c5dfe8637da","filter labels":["test-ns-thrrw",{"ray.io/cluster":"stop-z5q6j","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:48:25.967Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"186f1bc4-a6fb-4a11-a4f2-1c5dfe8637da","filter labels":["test-ns-thrrw",{"ray.io/cluster":"stop-z5q6j","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:48:25.967Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"186f1bc4-a6fb-4a11-a4f2-1c5dfe8637da","filter labels":["test-ns-thrrw",{"ray.io/cluster":"stop-z5q6j","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:48:25.967Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"186f1bc4-a6fb-4a11-a4f2-1c5dfe8637da"}
{"level":"error","ts":"2026-06-05T09:48:25.967Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"186f1bc4-a6fb-4a11-a4f2-1c5dfe8637da","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-z5q6j not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:26.288Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"fbf37c21-3725-4306-abcb-e0ed6efc62f0"}
{"level":"info","ts":"2026-06-05T09:48:26.288Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"fbf37c21-3725-4306-abcb-e0ed6efc62f0"}
{"level":"info","ts":"2026-06-05T09:48:26.288Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"fbf37c21-3725-4306-abcb-e0ed6efc62f0","1 head service found":"stop-z5q6j-head-svc"}
{"level":"info","ts":"2026-06-05T09:48:26.288Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"fbf37c21-3725-4306-abcb-e0ed6efc62f0","error":"head certificate secret ray-head-secret-stop-z5q6j not found"}
{"level":"error","ts":"2026-06-05T09:48:26.288Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"fbf37c21-3725-4306-abcb-e0ed6efc62f0","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-z5q6j not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:26.288Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"fbf37c21-3725-4306-abcb-e0ed6efc62f0","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-z5q6j not found"}
{"level":"info","ts":"2026-06-05T09:48:26.288Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"fbf37c21-3725-4306-abcb-e0ed6efc62f0","filter labels":["test-ns-thrrw",{"ray.io/cluster":"stop-z5q6j","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:48:26.288Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"fbf37c21-3725-4306-abcb-e0ed6efc62f0","filter labels":["test-ns-thrrw",{"ray.io/cluster":"stop-z5q6j","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:48:26.288Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"fbf37c21-3725-4306-abcb-e0ed6efc62f0","filter labels":["test-ns-thrrw",{"ray.io/cluster":"stop-z5q6j","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:48:26.288Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"fbf37c21-3725-4306-abcb-e0ed6efc62f0"}
{"level":"error","ts":"2026-06-05T09:48:26.288Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"fbf37c21-3725-4306-abcb-e0ed6efc62f0","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-z5q6j not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:26.930Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"38bc1580-f714-4b7a-acb7-3b5614609a9b"}
{"level":"info","ts":"2026-06-05T09:48:26.930Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"38bc1580-f714-4b7a-acb7-3b5614609a9b"}
{"level":"info","ts":"2026-06-05T09:48:26.930Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"38bc1580-f714-4b7a-acb7-3b5614609a9b","1 head service found":"stop-z5q6j-head-svc"}
{"level":"info","ts":"2026-06-05T09:48:26.930Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"38bc1580-f714-4b7a-acb7-3b5614609a9b","error":"head certificate secret ray-head-secret-stop-z5q6j not found"}
{"level":"error","ts":"2026-06-05T09:48:26.930Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"38bc1580-f714-4b7a-acb7-3b5614609a9b","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-z5q6j not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:26.930Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"38bc1580-f714-4b7a-acb7-3b5614609a9b","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-z5q6j not found"}
{"level":"info","ts":"2026-06-05T09:48:26.930Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"38bc1580-f714-4b7a-acb7-3b5614609a9b","filter labels":["test-ns-thrrw",{"ray.io/cluster":"stop-z5q6j","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:48:26.930Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"38bc1580-f714-4b7a-acb7-3b5614609a9b","filter labels":["test-ns-thrrw",{"ray.io/cluster":"stop-z5q6j","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:48:26.930Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"38bc1580-f714-4b7a-acb7-3b5614609a9b","filter labels":["test-ns-thrrw",{"ray.io/cluster":"stop-z5q6j","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:48:26.930Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"38bc1580-f714-4b7a-acb7-3b5614609a9b"}
{"level":"error","ts":"2026-06-05T09:48:26.930Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"38bc1580-f714-4b7a-acb7-3b5614609a9b","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-z5q6j not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:27.884Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-thrrw"},"reconcileID":"f1eddf9d-4f09-4676-aff7-824c41c11b47"}
{"level":"info","ts":"2026-06-05T09:48:28.210Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"b2999ccb-ee65-4790-a35d-6f9690f20a4b"}
{"level":"info","ts":"2026-06-05T09:48:28.210Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"b2999ccb-ee65-4790-a35d-6f9690f20a4b"}
{"level":"info","ts":"2026-06-05T09:48:28.210Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"b2999ccb-ee65-4790-a35d-6f9690f20a4b","1 head service found":"stop-z5q6j-head-svc"}
{"level":"info","ts":"2026-06-05T09:48:28.210Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"b2999ccb-ee65-4790-a35d-6f9690f20a4b"}
{"level":"info","ts":"2026-06-05T09:48:28.210Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"b2999ccb-ee65-4790-a35d-6f9690f20a4b","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:48:28.210Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"b2999ccb-ee65-4790-a35d-6f9690f20a4b"}
{"level":"info","ts":"2026-06-05T09:48:28.210Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"b2999ccb-ee65-4790-a35d-6f9690f20a4b","mode":"IntegratedOAuth","cluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:28.210Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"b2999ccb-ee65-4790-a35d-6f9690f20a4b","cluster":"stop-z5q6j","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:48:28.210Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"b2999ccb-ee65-4790-a35d-6f9690f20a4b","cluster":"stop-z5q6j","authType":"IntegratedOAuth","serviceAccount":"stop-z5q6j-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-05T09:48:28.210Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"b2999ccb-ee65-4790-a35d-6f9690f20a4b","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"stop-z5q6j","ray.io/group":"headgroup","ray.io/identifier":"stop-z5q6j-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-05T09:48:28.210Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"b2999ccb-ee65-4790-a35d-6f9690f20a4b","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"stop-z5q6j-head-svc.test-ns-thrrw.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-05T09:48:28.210Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"b2999ccb-ee65-4790-a35d-6f9690f20a4b","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=stop-z5q6j-head-svc.test-ns-thrrw.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-05T09:48:28.210Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"b2999ccb-ee65-4790-a35d-6f9690f20a4b","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=stop-z5q6j-head-svc.test-ns-thrrw.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-05T09:48:28.210Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"b2999ccb-ee65-4790-a35d-6f9690f20a4b","enabled":true}
{"level":"info","ts":"2026-06-05T09:48:28.228Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"b2999ccb-ee65-4790-a35d-6f9690f20a4b","name":"stop-z5q6j-head-kpbk9"}
{"level":"info","ts":"2026-06-05T09:48:28.228Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"b2999ccb-ee65-4790-a35d-6f9690f20a4b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:48:28.228Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"b2999ccb-ee65-4790-a35d-6f9690f20a4b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:48:28.228Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"b2999ccb-ee65-4790-a35d-6f9690f20a4b","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-05T09:48:28.228Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"b2999ccb-ee65-4790-a35d-6f9690f20a4b","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-05T09:48:28.228Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"b2999ccb-ee65-4790-a35d-6f9690f20a4b","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-05T09:48:28.228Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"b2999ccb-ee65-4790-a35d-6f9690f20a4b"}
{"level":"info","ts":"2026-06-05T09:48:28.228Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"b2999ccb-ee65-4790-a35d-6f9690f20a4b","nodeType":"worker","rayStartParams":{"address":"stop-z5q6j-head-svc.test-ns-thrrw.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-05T09:48:28.228Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"b2999ccb-ee65-4790-a35d-6f9690f20a4b","rayStartCmd":"ray start  --address=stop-z5q6j-head-svc.test-ns-thrrw.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:48:28.228Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"b2999ccb-ee65-4790-a35d-6f9690f20a4b","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=stop-z5q6j-head-svc.test-ns-thrrw.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:48:28.228Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"b2999ccb-ee65-4790-a35d-6f9690f20a4b","enabled":true}
{"level":"info","ts":"2026-06-05T09:48:28.242Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"b2999ccb-ee65-4790-a35d-6f9690f20a4b","name":"stop-z5q6j-small-group-worker-kjlsx"}
{"level":"info","ts":"2026-06-05T09:48:28.242Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"b2999ccb-ee65-4790-a35d-6f9690f20a4b","podName":"stop-z5q6j-head-kpbk9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:48:28.243Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"b2999ccb-ee65-4790-a35d-6f9690f20a4b","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"stop-z5q6j-head-svc"},"newHead":{"podName":"stop-z5q6j-head-kpbk9","serviceName":"stop-z5q6j-head-svc"}}
{"level":"info","ts":"2026-06-05T09:48:28.243Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"b2999ccb-ee65-4790-a35d-6f9690f20a4b","name":"stop-z5q6j","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:48:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-z5q6j-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:48:28Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-z5q6j-head-kpbk9","serviceName":"stop-z5q6j-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:48:28.257Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"8b33f8fc-0ea9-4271-8336-76e06b5abfe5","cluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:28.257Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"55089aaa-e4af-47ee-8629-d54e09d0f00e","rayCluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:28.257Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"55089aaa-e4af-47ee-8629-d54e09d0f00e","rayCluster":"stop-z5q6j","podIPs":[]}
{"level":"info","ts":"2026-06-05T09:48:28.257Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"9f3d1a49-76ae-4516-9272-a70c9aae2cb8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:48:28.257Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"9f3d1a49-76ae-4516-9272-a70c9aae2cb8","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:28.257Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"9f3d1a49-76ae-4516-9272-a70c9aae2cb8","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:48:28.257Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"9f3d1a49-76ae-4516-9272-a70c9aae2cb8"}
{"level":"info","ts":"2026-06-05T09:48:28.257Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"9f3d1a49-76ae-4516-9272-a70c9aae2cb8","RayCluster":"stop-z5q6j","State":""}
{"level":"info","ts":"2026-06-05T09:48:28.269Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"8b33f8fc-0ea9-4271-8336-76e06b5abfe5","name":"stop-z5q6j-head"}
{"level":"info","ts":"2026-06-05T09:48:28.305Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"8b33f8fc-0ea9-4271-8336-76e06b5abfe5","name":"stop-z5q6j-workers"}
{"level":"info","ts":"2026-06-05T09:48:28.306Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"8b33f8fc-0ea9-4271-8336-76e06b5abfe5","cluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:28.305Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"444a0f46-e207-465a-9651-2fa97c388c76"}
{"level":"info","ts":"2026-06-05T09:48:28.306Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"444a0f46-e207-465a-9651-2fa97c388c76"}
{"level":"info","ts":"2026-06-05T09:48:28.306Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"444a0f46-e207-465a-9651-2fa97c388c76","1 head service found":"stop-z5q6j-head-svc"}
{"level":"info","ts":"2026-06-05T09:48:28.306Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"444a0f46-e207-465a-9651-2fa97c388c76","Found 1 head Pod":"stop-z5q6j-head-kpbk9","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:48:28.306Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"444a0f46-e207-465a-9651-2fa97c388c76","head Pod":"stop-z5q6j-head-kpbk9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-z5q6j-head-kpbk9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:48:28.306Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"444a0f46-e207-465a-9651-2fa97c388c76","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:48:28.306Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"444a0f46-e207-465a-9651-2fa97c388c76","worker Pod":"stop-z5q6j-small-group-worker-kjlsx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-z5q6j-small-group-worker-kjlsx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:48:28.306Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"444a0f46-e207-465a-9651-2fa97c388c76","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:48:28.306Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"444a0f46-e207-465a-9651-2fa97c388c76","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:48:28.306Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"444a0f46-e207-465a-9651-2fa97c388c76","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:48:28.306Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"444a0f46-e207-465a-9651-2fa97c388c76","podName":"stop-z5q6j-head-kpbk9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:48:28.306Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"444a0f46-e207-465a-9651-2fa97c388c76","podName":"stop-z5q6j-head-kpbk9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:48:28.306Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"444a0f46-e207-465a-9651-2fa97c388c76","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-05T09:48:28.405Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"444a0f46-e207-465a-9651-2fa97c388c76","name":"stop-z5q6j","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:48:28Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-z5q6j-head-kpbk9","serviceName":"stop-z5q6j-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:48:28Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-z5q6j-head-kpbk9","serviceName":"stop-z5q6j-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:48:28.406Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"ee3d8691-94f3-480b-aafb-ab7b0e78756b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:48:28.406Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"ee3d8691-94f3-480b-aafb-ab7b0e78756b","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:28.406Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"ee3d8691-94f3-480b-aafb-ab7b0e78756b","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:48:28.407Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"ee3d8691-94f3-480b-aafb-ab7b0e78756b"}
{"level":"info","ts":"2026-06-05T09:48:28.407Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"ee3d8691-94f3-480b-aafb-ab7b0e78756b","RayCluster":"stop-z5q6j","State":""}
{"level":"info","ts":"2026-06-05T09:48:28.419Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"41258415-eaa6-404d-a385-43b561830cb0","rayCluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:28.420Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"9ecd14a0-a601-4e30-ac75-841f8a5cce4c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:48:28.420Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"9ecd14a0-a601-4e30-ac75-841f8a5cce4c","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:28.420Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"9ecd14a0-a601-4e30-ac75-841f8a5cce4c","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:48:28.420Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"41258415-eaa6-404d-a385-43b561830cb0","rayCluster":"stop-z5q6j","podIPs":[]}
{"level":"info","ts":"2026-06-05T09:48:28.419Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"6d24d13a-413a-42e0-bb71-9a40e491b6f3","cluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:28.420Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"9ecd14a0-a601-4e30-ac75-841f8a5cce4c"}
{"level":"info","ts":"2026-06-05T09:48:28.420Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"9ecd14a0-a601-4e30-ac75-841f8a5cce4c","RayCluster":"stop-z5q6j","State":""}
{"level":"info","ts":"2026-06-05T09:48:28.432Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"6d24d13a-413a-42e0-bb71-9a40e491b6f3","name":"stop-z5q6j-head"}
{"level":"info","ts":"2026-06-05T09:48:28.506Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"6d24d13a-413a-42e0-bb71-9a40e491b6f3","name":"stop-z5q6j-workers"}
{"level":"info","ts":"2026-06-05T09:48:28.506Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"6d24d13a-413a-42e0-bb71-9a40e491b6f3","cluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:28.681Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"755ef88a-febd-4417-b0d8-fdb6bdd023e4"}
{"level":"info","ts":"2026-06-05T09:48:28.681Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"755ef88a-febd-4417-b0d8-fdb6bdd023e4"}
{"level":"info","ts":"2026-06-05T09:48:28.681Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"755ef88a-febd-4417-b0d8-fdb6bdd023e4","1 head service found":"stop-z5q6j-head-svc"}
{"level":"info","ts":"2026-06-05T09:48:28.681Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"755ef88a-febd-4417-b0d8-fdb6bdd023e4","Found 1 head Pod":"stop-z5q6j-head-kpbk9","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:48:28.681Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"755ef88a-febd-4417-b0d8-fdb6bdd023e4","head Pod":"stop-z5q6j-head-kpbk9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-z5q6j-head-kpbk9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:48:28.681Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"755ef88a-febd-4417-b0d8-fdb6bdd023e4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:48:28.681Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"755ef88a-febd-4417-b0d8-fdb6bdd023e4","worker Pod":"stop-z5q6j-small-group-worker-kjlsx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-z5q6j-small-group-worker-kjlsx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:48:28.681Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"755ef88a-febd-4417-b0d8-fdb6bdd023e4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:48:28.681Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"755ef88a-febd-4417-b0d8-fdb6bdd023e4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:48:28.681Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"755ef88a-febd-4417-b0d8-fdb6bdd023e4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:48:28.681Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"755ef88a-febd-4417-b0d8-fdb6bdd023e4","podName":"stop-z5q6j-head-kpbk9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:48:28.681Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"755ef88a-febd-4417-b0d8-fdb6bdd023e4","podName":"stop-z5q6j-head-kpbk9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:48:28.682Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"755ef88a-febd-4417-b0d8-fdb6bdd023e4","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:48:28.682Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"755ef88a-febd-4417-b0d8-fdb6bdd023e4","seconds":300}
{"level":"info","ts":"2026-06-05T09:48:28.708Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"bcc8e67c-359e-40a3-b94a-d44d81b7dc56"}
{"level":"info","ts":"2026-06-05T09:48:28.708Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"bcc8e67c-359e-40a3-b94a-d44d81b7dc56"}
{"level":"info","ts":"2026-06-05T09:48:28.708Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"bcc8e67c-359e-40a3-b94a-d44d81b7dc56","1 head service found":"stop-z5q6j-head-svc"}
{"level":"info","ts":"2026-06-05T09:48:28.708Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"bcc8e67c-359e-40a3-b94a-d44d81b7dc56","Found 1 head Pod":"stop-z5q6j-head-kpbk9","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:48:28.708Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"bcc8e67c-359e-40a3-b94a-d44d81b7dc56","head Pod":"stop-z5q6j-head-kpbk9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-z5q6j-head-kpbk9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:48:28.708Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"bcc8e67c-359e-40a3-b94a-d44d81b7dc56","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:48:28.708Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"bcc8e67c-359e-40a3-b94a-d44d81b7dc56","worker Pod":"stop-z5q6j-small-group-worker-kjlsx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-z5q6j-small-group-worker-kjlsx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:48:28.708Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"bcc8e67c-359e-40a3-b94a-d44d81b7dc56","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:48:28.708Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"bcc8e67c-359e-40a3-b94a-d44d81b7dc56","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:48:28.708Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"bcc8e67c-359e-40a3-b94a-d44d81b7dc56","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:48:28.708Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"bcc8e67c-359e-40a3-b94a-d44d81b7dc56","podName":"stop-z5q6j-head-kpbk9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:48:28.708Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"bcc8e67c-359e-40a3-b94a-d44d81b7dc56","podName":"stop-z5q6j-head-kpbk9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:48:28.708Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"bcc8e67c-359e-40a3-b94a-d44d81b7dc56","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:48:28.708Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"bcc8e67c-359e-40a3-b94a-d44d81b7dc56","seconds":300}
{"level":"info","ts":"2026-06-05T09:48:28.887Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"4ace5774-8cd8-4779-9026-6b6aea2604ec"}
{"level":"info","ts":"2026-06-05T09:48:28.887Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"4ace5774-8cd8-4779-9026-6b6aea2604ec"}
{"level":"info","ts":"2026-06-05T09:48:28.887Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"4ace5774-8cd8-4779-9026-6b6aea2604ec","1 head service found":"stop-z5q6j-head-svc"}
{"level":"info","ts":"2026-06-05T09:48:28.887Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"4ace5774-8cd8-4779-9026-6b6aea2604ec","Found 1 head Pod":"stop-z5q6j-head-kpbk9","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:48:28.887Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"4ace5774-8cd8-4779-9026-6b6aea2604ec","head Pod":"stop-z5q6j-head-kpbk9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-z5q6j-head-kpbk9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:48:28.887Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"4ace5774-8cd8-4779-9026-6b6aea2604ec","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:48:28.887Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"4ace5774-8cd8-4779-9026-6b6aea2604ec","worker Pod":"stop-z5q6j-small-group-worker-kjlsx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-z5q6j-small-group-worker-kjlsx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:48:28.887Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"4ace5774-8cd8-4779-9026-6b6aea2604ec","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:48:28.887Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"4ace5774-8cd8-4779-9026-6b6aea2604ec","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:48:28.887Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"4ace5774-8cd8-4779-9026-6b6aea2604ec","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:48:28.888Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"4ace5774-8cd8-4779-9026-6b6aea2604ec","podName":"stop-z5q6j-head-kpbk9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:48:28.888Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"4ace5774-8cd8-4779-9026-6b6aea2604ec","podName":"stop-z5q6j-head-kpbk9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:48:28.888Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"4ace5774-8cd8-4779-9026-6b6aea2604ec","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:48:28.888Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"4ace5774-8cd8-4779-9026-6b6aea2604ec","seconds":300}
{"level":"info","ts":"2026-06-05T09:48:29.709Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"c6a1fabd-f122-4daf-9add-cd08e42445be"}
{"level":"info","ts":"2026-06-05T09:48:29.709Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"c6a1fabd-f122-4daf-9add-cd08e42445be"}
{"level":"info","ts":"2026-06-05T09:48:29.709Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"c6a1fabd-f122-4daf-9add-cd08e42445be","1 head service found":"stop-z5q6j-head-svc"}
{"level":"info","ts":"2026-06-05T09:48:29.709Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"c6a1fabd-f122-4daf-9add-cd08e42445be","Found 1 head Pod":"stop-z5q6j-head-kpbk9","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:48:29.709Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"c6a1fabd-f122-4daf-9add-cd08e42445be","head Pod":"stop-z5q6j-head-kpbk9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-z5q6j-head-kpbk9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:48:29.709Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"c6a1fabd-f122-4daf-9add-cd08e42445be","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:48:29.709Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"c6a1fabd-f122-4daf-9add-cd08e42445be","worker Pod":"stop-z5q6j-small-group-worker-kjlsx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-z5q6j-small-group-worker-kjlsx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:48:29.709Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"c6a1fabd-f122-4daf-9add-cd08e42445be","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:48:29.709Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"c6a1fabd-f122-4daf-9add-cd08e42445be","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:48:29.709Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"c6a1fabd-f122-4daf-9add-cd08e42445be","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:48:29.709Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"c6a1fabd-f122-4daf-9add-cd08e42445be","podName":"stop-z5q6j-head-kpbk9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:48:29.709Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"c6a1fabd-f122-4daf-9add-cd08e42445be","podName":"stop-z5q6j-head-kpbk9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:48:29.709Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"c6a1fabd-f122-4daf-9add-cd08e42445be","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"stop-z5q6j-head-kpbk9","serviceName":"stop-z5q6j-head-svc"},"newHead":{"podIP":"10.132.0.25","serviceIP":"10.132.0.25","podName":"stop-z5q6j-head-kpbk9","serviceName":"stop-z5q6j-head-svc"}}
{"level":"info","ts":"2026-06-05T09:48:29.709Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"c6a1fabd-f122-4daf-9add-cd08e42445be","name":"stop-z5q6j","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:48:28Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-z5q6j-head-kpbk9","serviceName":"stop-z5q6j-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:48:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.25","serviceIP":"10.132.0.25","podName":"stop-z5q6j-head-kpbk9","serviceName":"stop-z5q6j-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:48:29.722Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"98b47285-acc0-472c-9bac-bd77d2e63b2f","rayCluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:29.722Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"abfd0fba-6ac9-4c74-830b-dfaeab3954c2","cluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:29.722Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"aa5556bc-3b22-4d44-a734-47c7fd86daea","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:48:29.722Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"98b47285-acc0-472c-9bac-bd77d2e63b2f","pod":"stop-z5q6j-head-kpbk9","ip":"10.132.0.25"}
{"level":"info","ts":"2026-06-05T09:48:29.722Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"98b47285-acc0-472c-9bac-bd77d2e63b2f","pod":"stop-z5q6j-small-group-worker-kjlsx","ip":"10.133.0.51"}
{"level":"info","ts":"2026-06-05T09:48:29.722Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"aa5556bc-3b22-4d44-a734-47c7fd86daea","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:29.722Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"aa5556bc-3b22-4d44-a734-47c7fd86daea","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:48:29.722Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"aa5556bc-3b22-4d44-a734-47c7fd86daea"}
{"level":"info","ts":"2026-06-05T09:48:29.722Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"aa5556bc-3b22-4d44-a734-47c7fd86daea","RayCluster":"stop-z5q6j","State":""}
{"level":"info","ts":"2026-06-05T09:48:29.734Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"98b47285-acc0-472c-9bac-bd77d2e63b2f","certificate":"ray-head-cert-stop-z5q6j","dnsNames":["stop-z5q6j-head-svc","localhost","stop-z5q6j-head-svc.test-ns-thrrw.svc","stop-z5q6j-head-svc.test-ns-thrrw.svc.cluster.local"],"ipAddresses":["10.132.0.25","10.133.0.51","127.0.0.1"]}
{"level":"info","ts":"2026-06-05T09:48:29.742Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"abfd0fba-6ac9-4c74-830b-dfaeab3954c2","name":"stop-z5q6j-head"}
{"level":"info","ts":"2026-06-05T09:48:29.747Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"98b47285-acc0-472c-9bac-bd77d2e63b2f","certificate":"ray-worker-cert-stop-z5q6j","dnsNames":["stop-z5q6j-worker-svc","localhost","stop-z5q6j-small-group","stop-z5q6j-worker-svc.test-ns-thrrw.svc","stop-z5q6j-worker-svc.test-ns-thrrw.svc.cluster.local","*.stop-z5q6j-worker-svc.test-ns-thrrw.svc","*.stop-z5q6j-worker-svc.test-ns-thrrw.svc.cluster.local","*-worker-*.test-ns-thrrw.svc","*-worker-*.test-ns-thrrw.svc.cluster.local"],"ipAddresses":["10.132.0.25","10.133.0.51","127.0.0.1"]}
{"level":"info","ts":"2026-06-05T09:48:29.747Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"98b47285-acc0-472c-9bac-bd77d2e63b2f","rayCluster":"stop-z5q6j","podIPs":["10.132.0.25","10.133.0.51"]}
{"level":"info","ts":"2026-06-05T09:48:29.756Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"abfd0fba-6ac9-4c74-830b-dfaeab3954c2","name":"stop-z5q6j-workers"}
{"level":"info","ts":"2026-06-05T09:48:29.756Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"abfd0fba-6ac9-4c74-830b-dfaeab3954c2","cluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:30.306Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"916821ea-a15a-4cd9-ac3a-17e70895532a"}
{"level":"info","ts":"2026-06-05T09:48:30.306Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"916821ea-a15a-4cd9-ac3a-17e70895532a"}
{"level":"info","ts":"2026-06-05T09:48:30.306Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"916821ea-a15a-4cd9-ac3a-17e70895532a","1 head service found":"stop-z5q6j-head-svc"}
{"level":"info","ts":"2026-06-05T09:48:30.307Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"916821ea-a15a-4cd9-ac3a-17e70895532a","Found 1 head Pod":"stop-z5q6j-head-kpbk9","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:48:30.307Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"916821ea-a15a-4cd9-ac3a-17e70895532a","head Pod":"stop-z5q6j-head-kpbk9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-z5q6j-head-kpbk9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:48:30.307Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"916821ea-a15a-4cd9-ac3a-17e70895532a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:48:30.307Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"916821ea-a15a-4cd9-ac3a-17e70895532a","worker Pod":"stop-z5q6j-small-group-worker-kjlsx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-z5q6j-small-group-worker-kjlsx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:48:30.307Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"916821ea-a15a-4cd9-ac3a-17e70895532a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:48:30.307Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"916821ea-a15a-4cd9-ac3a-17e70895532a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:48:30.307Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"916821ea-a15a-4cd9-ac3a-17e70895532a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:48:30.307Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"916821ea-a15a-4cd9-ac3a-17e70895532a","podName":"stop-z5q6j-small-group-worker-kjlsx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:48:30.307Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"916821ea-a15a-4cd9-ac3a-17e70895532a","podName":"stop-z5q6j-small-group-worker-kjlsx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:48:30.307Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"916821ea-a15a-4cd9-ac3a-17e70895532a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:48:30.307Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"916821ea-a15a-4cd9-ac3a-17e70895532a","seconds":300}
{"level":"info","ts":"2026-06-05T09:48:31.408Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"ec9f6f51-6ff8-45b9-9b65-71772ba08208","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:48:31.408Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"ec9f6f51-6ff8-45b9-9b65-71772ba08208","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:31.408Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"ec9f6f51-6ff8-45b9-9b65-71772ba08208","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:48:31.408Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"ec9f6f51-6ff8-45b9-9b65-71772ba08208"}
{"level":"info","ts":"2026-06-05T09:48:31.408Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"ec9f6f51-6ff8-45b9-9b65-71772ba08208","RayCluster":"stop-z5q6j","State":""}
{"level":"info","ts":"2026-06-05T09:48:34.408Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"58e0731f-35d2-4d9a-8b00-5296058a6a97","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:48:34.408Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"58e0731f-35d2-4d9a-8b00-5296058a6a97","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:34.408Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"58e0731f-35d2-4d9a-8b00-5296058a6a97","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:48:34.409Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"58e0731f-35d2-4d9a-8b00-5296058a6a97"}
{"level":"info","ts":"2026-06-05T09:48:34.409Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"58e0731f-35d2-4d9a-8b00-5296058a6a97","RayCluster":"stop-z5q6j","State":""}
{"level":"info","ts":"2026-06-05T09:48:34.725Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"76171216-1d80-408e-9363-4cdcdd6152ec"}
{"level":"info","ts":"2026-06-05T09:48:34.725Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"76171216-1d80-408e-9363-4cdcdd6152ec"}
{"level":"info","ts":"2026-06-05T09:48:34.726Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"76171216-1d80-408e-9363-4cdcdd6152ec","1 head service found":"stop-z5q6j-head-svc"}
{"level":"info","ts":"2026-06-05T09:48:34.726Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"76171216-1d80-408e-9363-4cdcdd6152ec","Found 1 head Pod":"stop-z5q6j-head-kpbk9","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:48:34.726Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"76171216-1d80-408e-9363-4cdcdd6152ec","head Pod":"stop-z5q6j-head-kpbk9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-z5q6j-head-kpbk9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:48:34.726Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"76171216-1d80-408e-9363-4cdcdd6152ec","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:48:34.726Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"76171216-1d80-408e-9363-4cdcdd6152ec","worker Pod":"stop-z5q6j-small-group-worker-kjlsx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-z5q6j-small-group-worker-kjlsx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:48:34.726Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"76171216-1d80-408e-9363-4cdcdd6152ec","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:48:34.726Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"76171216-1d80-408e-9363-4cdcdd6152ec","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:48:34.726Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"76171216-1d80-408e-9363-4cdcdd6152ec","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:48:34.726Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"76171216-1d80-408e-9363-4cdcdd6152ec","podName":"stop-z5q6j-head-kpbk9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:48:34.726Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"76171216-1d80-408e-9363-4cdcdd6152ec","podName":"stop-z5q6j-head-kpbk9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:48:34.726Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"76171216-1d80-408e-9363-4cdcdd6152ec","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:48:34.726Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"76171216-1d80-408e-9363-4cdcdd6152ec","seconds":300}
{"level":"info","ts":"2026-06-05T09:48:35.731Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"3bec991e-f4fa-4577-8683-5b1385c2847a"}
{"level":"info","ts":"2026-06-05T09:48:35.731Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"3bec991e-f4fa-4577-8683-5b1385c2847a"}
{"level":"info","ts":"2026-06-05T09:48:35.731Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"3bec991e-f4fa-4577-8683-5b1385c2847a","1 head service found":"stop-z5q6j-head-svc"}
{"level":"info","ts":"2026-06-05T09:48:35.731Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"3bec991e-f4fa-4577-8683-5b1385c2847a","Found 1 head Pod":"stop-z5q6j-head-kpbk9","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:48:35.731Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"3bec991e-f4fa-4577-8683-5b1385c2847a","head Pod":"stop-z5q6j-head-kpbk9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-z5q6j-head-kpbk9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:48:35.731Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"3bec991e-f4fa-4577-8683-5b1385c2847a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:48:35.731Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"3bec991e-f4fa-4577-8683-5b1385c2847a","worker Pod":"stop-z5q6j-small-group-worker-kjlsx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-z5q6j-small-group-worker-kjlsx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:48:35.731Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"3bec991e-f4fa-4577-8683-5b1385c2847a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:48:35.731Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"3bec991e-f4fa-4577-8683-5b1385c2847a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:48:35.731Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"3bec991e-f4fa-4577-8683-5b1385c2847a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:48:35.731Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"3bec991e-f4fa-4577-8683-5b1385c2847a","podName":"stop-z5q6j-head-kpbk9","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:48:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:48:35.731Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"3bec991e-f4fa-4577-8683-5b1385c2847a","podName":"stop-z5q6j-head-kpbk9","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:48:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:48:35.731Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"3bec991e-f4fa-4577-8683-5b1385c2847a","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-05T09:48:35.731Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"3bec991e-f4fa-4577-8683-5b1385c2847a","name":"stop-z5q6j","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:48:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.25","serviceIP":"10.132.0.25","podName":"stop-z5q6j-head-kpbk9","serviceName":"stop-z5q6j-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:48:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.25","serviceIP":"10.132.0.25","podName":"stop-z5q6j-head-kpbk9","serviceName":"stop-z5q6j-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:48:35.745Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"7bd0d74d-ec6e-4aa2-8546-fa6c8782f06c","rayCluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:35.745Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"0b24c658-e0c3-4be5-87c2-2aa33cb5f604","cluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:35.745Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"7bd0d74d-ec6e-4aa2-8546-fa6c8782f06c","pod":"stop-z5q6j-head-kpbk9","ip":"10.132.0.25"}
{"level":"info","ts":"2026-06-05T09:48:35.745Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"7bd0d74d-ec6e-4aa2-8546-fa6c8782f06c","pod":"stop-z5q6j-small-group-worker-kjlsx","ip":"10.133.0.51"}
{"level":"info","ts":"2026-06-05T09:48:35.745Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"77324652-1efa-4c4b-b0fa-949f3a0a0f73","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:48:35.745Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"7bd0d74d-ec6e-4aa2-8546-fa6c8782f06c","rayCluster":"stop-z5q6j","podIPs":["10.132.0.25","10.133.0.51"]}
{"level":"info","ts":"2026-06-05T09:48:35.745Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"77324652-1efa-4c4b-b0fa-949f3a0a0f73","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:35.745Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"77324652-1efa-4c4b-b0fa-949f3a0a0f73","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:48:35.745Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"77324652-1efa-4c4b-b0fa-949f3a0a0f73"}
{"level":"info","ts":"2026-06-05T09:48:35.745Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"77324652-1efa-4c4b-b0fa-949f3a0a0f73","RayCluster":"stop-z5q6j","State":""}
{"level":"info","ts":"2026-06-05T09:48:35.755Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"0b24c658-e0c3-4be5-87c2-2aa33cb5f604","name":"stop-z5q6j-head"}
{"level":"info","ts":"2026-06-05T09:48:35.767Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"0b24c658-e0c3-4be5-87c2-2aa33cb5f604","name":"stop-z5q6j-workers"}
{"level":"info","ts":"2026-06-05T09:48:35.767Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"0b24c658-e0c3-4be5-87c2-2aa33cb5f604","cluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:36.556Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"9159af74-24ab-46cf-b3e8-92005b369444","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:48:36.557Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"9159af74-24ab-46cf-b3e8-92005b369444","pod":"raycluster-small-group-worker-j9pbw","ip":"10.133.0.35"}
{"level":"info","ts":"2026-06-05T09:48:36.557Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"9159af74-24ab-46cf-b3e8-92005b369444","pod":"raycluster-head-vjd7d","ip":"10.134.0.20"}
{"level":"info","ts":"2026-06-05T09:48:36.557Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"9159af74-24ab-46cf-b3e8-92005b369444","rayCluster":"raycluster","podIPs":["10.133.0.35","10.134.0.20"]}
{"level":"info","ts":"2026-06-05T09:48:37.409Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"9fbc58bb-ee22-47f1-b02f-892a2e421fd2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:48:37.409Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"9fbc58bb-ee22-47f1-b02f-892a2e421fd2","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:37.409Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"9fbc58bb-ee22-47f1-b02f-892a2e421fd2","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:48:37.409Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"9fbc58bb-ee22-47f1-b02f-892a2e421fd2"}
{"level":"info","ts":"2026-06-05T09:48:37.409Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"9fbc58bb-ee22-47f1-b02f-892a2e421fd2","RayCluster":"stop-z5q6j","State":""}
{"level":"info","ts":"2026-06-05T09:48:37.745Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"074ff42f-7fe1-45a7-a4c7-f09ff164c6ac"}
{"level":"info","ts":"2026-06-05T09:48:37.745Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"074ff42f-7fe1-45a7-a4c7-f09ff164c6ac"}
{"level":"info","ts":"2026-06-05T09:48:37.745Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"074ff42f-7fe1-45a7-a4c7-f09ff164c6ac","1 head service found":"stop-z5q6j-head-svc"}
{"level":"info","ts":"2026-06-05T09:48:37.745Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"074ff42f-7fe1-45a7-a4c7-f09ff164c6ac","Found 1 head Pod":"stop-z5q6j-head-kpbk9","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:48:37.746Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"074ff42f-7fe1-45a7-a4c7-f09ff164c6ac","head Pod":"stop-z5q6j-head-kpbk9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-z5q6j-head-kpbk9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:48:37.746Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"074ff42f-7fe1-45a7-a4c7-f09ff164c6ac","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:48:37.746Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"074ff42f-7fe1-45a7-a4c7-f09ff164c6ac","worker Pod":"stop-z5q6j-small-group-worker-kjlsx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-z5q6j-small-group-worker-kjlsx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:48:37.746Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"074ff42f-7fe1-45a7-a4c7-f09ff164c6ac","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:48:37.746Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"074ff42f-7fe1-45a7-a4c7-f09ff164c6ac","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:48:37.746Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"074ff42f-7fe1-45a7-a4c7-f09ff164c6ac","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:48:37.746Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"074ff42f-7fe1-45a7-a4c7-f09ff164c6ac","podName":"stop-z5q6j-head-kpbk9","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:48:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:48:37.746Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"074ff42f-7fe1-45a7-a4c7-f09ff164c6ac","podName":"stop-z5q6j-head-kpbk9","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:48:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:48:37.746Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"074ff42f-7fe1-45a7-a4c7-f09ff164c6ac","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:48:37.746Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"074ff42f-7fe1-45a7-a4c7-f09ff164c6ac","seconds":300}
{"level":"info","ts":"2026-06-05T09:48:38.940Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"51faebff-3c41-4d1e-b6a6-225b44d9fc12"}
{"level":"info","ts":"2026-06-05T09:48:38.940Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"51faebff-3c41-4d1e-b6a6-225b44d9fc12"}
{"level":"info","ts":"2026-06-05T09:48:38.940Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"51faebff-3c41-4d1e-b6a6-225b44d9fc12","1 head service found":"stop-z5q6j-head-svc"}
{"level":"info","ts":"2026-06-05T09:48:38.941Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"51faebff-3c41-4d1e-b6a6-225b44d9fc12","Found 1 head Pod":"stop-z5q6j-head-kpbk9","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:48:38.941Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"51faebff-3c41-4d1e-b6a6-225b44d9fc12","head Pod":"stop-z5q6j-head-kpbk9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-z5q6j-head-kpbk9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:48:38.941Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"51faebff-3c41-4d1e-b6a6-225b44d9fc12","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:48:38.941Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"51faebff-3c41-4d1e-b6a6-225b44d9fc12","worker Pod":"stop-z5q6j-small-group-worker-kjlsx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-z5q6j-small-group-worker-kjlsx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:48:38.941Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"51faebff-3c41-4d1e-b6a6-225b44d9fc12","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:48:38.941Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"51faebff-3c41-4d1e-b6a6-225b44d9fc12","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:48:38.941Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"51faebff-3c41-4d1e-b6a6-225b44d9fc12","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:48:38.941Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"51faebff-3c41-4d1e-b6a6-225b44d9fc12","podName":"stop-z5q6j-head-kpbk9","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:48:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:48:38.941Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"51faebff-3c41-4d1e-b6a6-225b44d9fc12","podName":"stop-z5q6j-head-kpbk9","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:48:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:48:38.941Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"51faebff-3c41-4d1e-b6a6-225b44d9fc12","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:48:38.941Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"51faebff-3c41-4d1e-b6a6-225b44d9fc12","seconds":300}
{"level":"info","ts":"2026-06-05T09:48:39.948Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"ea71eb79-48e1-4096-abd7-5bb0aa7f932b"}
{"level":"info","ts":"2026-06-05T09:48:39.949Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"ea71eb79-48e1-4096-abd7-5bb0aa7f932b"}
{"level":"info","ts":"2026-06-05T09:48:39.949Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"ea71eb79-48e1-4096-abd7-5bb0aa7f932b","1 head service found":"stop-z5q6j-head-svc"}
{"level":"info","ts":"2026-06-05T09:48:39.949Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"ea71eb79-48e1-4096-abd7-5bb0aa7f932b","Found 1 head Pod":"stop-z5q6j-head-kpbk9","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:48:39.949Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"ea71eb79-48e1-4096-abd7-5bb0aa7f932b","head Pod":"stop-z5q6j-head-kpbk9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-z5q6j-head-kpbk9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:48:39.949Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"ea71eb79-48e1-4096-abd7-5bb0aa7f932b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:48:39.949Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"ea71eb79-48e1-4096-abd7-5bb0aa7f932b","worker Pod":"stop-z5q6j-small-group-worker-kjlsx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-z5q6j-small-group-worker-kjlsx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:48:39.949Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"ea71eb79-48e1-4096-abd7-5bb0aa7f932b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:48:39.949Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"ea71eb79-48e1-4096-abd7-5bb0aa7f932b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:48:39.949Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"ea71eb79-48e1-4096-abd7-5bb0aa7f932b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:48:39.949Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"ea71eb79-48e1-4096-abd7-5bb0aa7f932b","podName":"stop-z5q6j-head-kpbk9","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:48:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:48:39.949Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"ea71eb79-48e1-4096-abd7-5bb0aa7f932b","podName":"stop-z5q6j-head-kpbk9","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:48:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:48:39.949Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"ea71eb79-48e1-4096-abd7-5bb0aa7f932b","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-05T09:48:39.949Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"ea71eb79-48e1-4096-abd7-5bb0aa7f932b","name":"stop-z5q6j","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:48:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.25","serviceIP":"10.132.0.25","podName":"stop-z5q6j-head-kpbk9","serviceName":"stop-z5q6j-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:48:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.25","serviceIP":"10.132.0.25","podName":"stop-z5q6j-head-kpbk9","serviceName":"stop-z5q6j-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:48:39.961Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"ae3f6c1a-0883-4252-858e-7e18c199984d","cluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:39.961Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"acc80a31-2198-4b77-a208-b7bf1211b2cd","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:48:39.961Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"acc80a31-2198-4b77-a208-b7bf1211b2cd","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:39.961Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"acc80a31-2198-4b77-a208-b7bf1211b2cd","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:48:39.961Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"9253bb73-1530-4740-9abc-fca16a7a3109","rayCluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:39.961Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"acc80a31-2198-4b77-a208-b7bf1211b2cd"}
{"level":"info","ts":"2026-06-05T09:48:39.961Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"acc80a31-2198-4b77-a208-b7bf1211b2cd","RayCluster":"stop-z5q6j","State":""}
{"level":"info","ts":"2026-06-05T09:48:39.961Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"9253bb73-1530-4740-9abc-fca16a7a3109","pod":"stop-z5q6j-small-group-worker-kjlsx","ip":"10.133.0.51"}
{"level":"info","ts":"2026-06-05T09:48:39.961Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"9253bb73-1530-4740-9abc-fca16a7a3109","pod":"stop-z5q6j-head-kpbk9","ip":"10.132.0.25"}
{"level":"info","ts":"2026-06-05T09:48:39.961Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"9253bb73-1530-4740-9abc-fca16a7a3109","rayCluster":"stop-z5q6j","podIPs":["10.133.0.51","10.132.0.25"]}
{"level":"info","ts":"2026-06-05T09:48:39.974Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"ae3f6c1a-0883-4252-858e-7e18c199984d","name":"stop-z5q6j-head"}
{"level":"info","ts":"2026-06-05T09:48:39.984Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"ae3f6c1a-0883-4252-858e-7e18c199984d","name":"stop-z5q6j-workers"}
{"level":"info","ts":"2026-06-05T09:48:39.984Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"ae3f6c1a-0883-4252-858e-7e18c199984d","cluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:40.410Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"83c1b471-99b4-4dae-a354-9e3a34f39e91","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:48:40.410Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"83c1b471-99b4-4dae-a354-9e3a34f39e91","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:40.410Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"83c1b471-99b4-4dae-a354-9e3a34f39e91","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:48:40.410Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"83c1b471-99b4-4dae-a354-9e3a34f39e91"}
{"level":"info","ts":"2026-06-05T09:48:40.410Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"83c1b471-99b4-4dae-a354-9e3a34f39e91","RayCluster":"stop-z5q6j","State":""}
{"level":"info","ts":"2026-06-05T09:48:41.961Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"74a406f3-65d8-4c24-9c6f-6041ad288b0b"}
{"level":"info","ts":"2026-06-05T09:48:41.961Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"74a406f3-65d8-4c24-9c6f-6041ad288b0b"}
{"level":"info","ts":"2026-06-05T09:48:41.962Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"74a406f3-65d8-4c24-9c6f-6041ad288b0b","1 head service found":"stop-z5q6j-head-svc"}
{"level":"info","ts":"2026-06-05T09:48:41.962Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"74a406f3-65d8-4c24-9c6f-6041ad288b0b","Found 1 head Pod":"stop-z5q6j-head-kpbk9","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:48:41.962Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"74a406f3-65d8-4c24-9c6f-6041ad288b0b","head Pod":"stop-z5q6j-head-kpbk9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-z5q6j-head-kpbk9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:48:41.962Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"74a406f3-65d8-4c24-9c6f-6041ad288b0b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:48:41.962Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"74a406f3-65d8-4c24-9c6f-6041ad288b0b","worker Pod":"stop-z5q6j-small-group-worker-kjlsx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-z5q6j-small-group-worker-kjlsx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:48:41.962Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"74a406f3-65d8-4c24-9c6f-6041ad288b0b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:48:41.962Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"74a406f3-65d8-4c24-9c6f-6041ad288b0b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:48:41.962Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"74a406f3-65d8-4c24-9c6f-6041ad288b0b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:48:41.962Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"74a406f3-65d8-4c24-9c6f-6041ad288b0b","podName":"stop-z5q6j-head-kpbk9","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:48:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:48:41.962Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"74a406f3-65d8-4c24-9c6f-6041ad288b0b","podName":"stop-z5q6j-head-kpbk9","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:48:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:48:41.962Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"74a406f3-65d8-4c24-9c6f-6041ad288b0b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:48:41.962Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"74a406f3-65d8-4c24-9c6f-6041ad288b0b","seconds":300}
{"level":"info","ts":"2026-06-05T09:48:43.410Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"bd2caaf8-b645-4efd-b5fa-51eb884c1b83","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:48:43.410Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"bd2caaf8-b645-4efd-b5fa-51eb884c1b83","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:43.410Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"bd2caaf8-b645-4efd-b5fa-51eb884c1b83","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:48:43.410Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"bd2caaf8-b645-4efd-b5fa-51eb884c1b83"}
{"level":"info","ts":"2026-06-05T09:48:43.410Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"bd2caaf8-b645-4efd-b5fa-51eb884c1b83","RayCluster":"stop-z5q6j","State":""}
{"level":"info","ts":"2026-06-05T09:48:46.411Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"edb0b6c9-08c3-4964-9bb1-6d09471f56b3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:48:46.411Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"edb0b6c9-08c3-4964-9bb1-6d09471f56b3","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:46.411Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"edb0b6c9-08c3-4964-9bb1-6d09471f56b3","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:48:46.411Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"edb0b6c9-08c3-4964-9bb1-6d09471f56b3"}
{"level":"info","ts":"2026-06-05T09:48:46.411Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"edb0b6c9-08c3-4964-9bb1-6d09471f56b3","RayCluster":"stop-z5q6j","State":""}
{"level":"info","ts":"2026-06-05T09:48:46.767Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"a4ec9652-fa8d-41f9-a54f-bc9fca4f2c50"}
{"level":"info","ts":"2026-06-05T09:48:46.767Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"a4ec9652-fa8d-41f9-a54f-bc9fca4f2c50"}
{"level":"info","ts":"2026-06-05T09:48:46.767Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"a4ec9652-fa8d-41f9-a54f-bc9fca4f2c50","1 head service found":"stop-z5q6j-head-svc"}
{"level":"info","ts":"2026-06-05T09:48:46.767Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"a4ec9652-fa8d-41f9-a54f-bc9fca4f2c50","Found 1 head Pod":"stop-z5q6j-head-kpbk9","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:48:46.767Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"a4ec9652-fa8d-41f9-a54f-bc9fca4f2c50","head Pod":"stop-z5q6j-head-kpbk9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-z5q6j-head-kpbk9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:48:46.767Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"a4ec9652-fa8d-41f9-a54f-bc9fca4f2c50","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:48:46.767Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"a4ec9652-fa8d-41f9-a54f-bc9fca4f2c50","worker Pod":"stop-z5q6j-small-group-worker-kjlsx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-z5q6j-small-group-worker-kjlsx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:48:46.767Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"a4ec9652-fa8d-41f9-a54f-bc9fca4f2c50","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:48:46.767Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"a4ec9652-fa8d-41f9-a54f-bc9fca4f2c50","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:48:46.767Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"a4ec9652-fa8d-41f9-a54f-bc9fca4f2c50","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:48:46.767Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"a4ec9652-fa8d-41f9-a54f-bc9fca4f2c50","podName":"stop-z5q6j-small-group-worker-kjlsx","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:48:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-05T09:48:46.767Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"a4ec9652-fa8d-41f9-a54f-bc9fca4f2c50","podName":"stop-z5q6j-small-group-worker-kjlsx","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:48:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-05T09:48:46.767Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"a4ec9652-fa8d-41f9-a54f-bc9fca4f2c50","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:48:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-05T09:48:46.767Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"a4ec9652-fa8d-41f9-a54f-bc9fca4f2c50","name":"stop-z5q6j","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:48:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.25","serviceIP":"10.132.0.25","podName":"stop-z5q6j-head-kpbk9","serviceName":"stop-z5q6j-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:48:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.25","serviceIP":"10.132.0.25","podName":"stop-z5q6j-head-kpbk9","serviceName":"stop-z5q6j-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:48:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:48:46.780Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"ee715ea9-7688-42ed-b4be-97ed7d94cd8f","rayCluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:46.780Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"9fc57c91-cc49-4017-8bd1-b1b00fea8721","cluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:46.780Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"d08f51fd-dc7e-4e09-b56b-fccbb26ef29d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:48:46.780Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"d08f51fd-dc7e-4e09-b56b-fccbb26ef29d","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:46.780Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"ee715ea9-7688-42ed-b4be-97ed7d94cd8f","pod":"stop-z5q6j-head-kpbk9","ip":"10.132.0.25"}
{"level":"info","ts":"2026-06-05T09:48:46.780Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"ee715ea9-7688-42ed-b4be-97ed7d94cd8f","pod":"stop-z5q6j-small-group-worker-kjlsx","ip":"10.133.0.51"}
{"level":"info","ts":"2026-06-05T09:48:46.780Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"d08f51fd-dc7e-4e09-b56b-fccbb26ef29d","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:48:46.780Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"ee715ea9-7688-42ed-b4be-97ed7d94cd8f","rayCluster":"stop-z5q6j","podIPs":["10.132.0.25","10.133.0.51"]}
{"level":"info","ts":"2026-06-05T09:48:46.781Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"d08f51fd-dc7e-4e09-b56b-fccbb26ef29d"}
{"level":"info","ts":"2026-06-05T09:48:46.781Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"d08f51fd-dc7e-4e09-b56b-fccbb26ef29d","RayCluster":"stop-z5q6j","State":""}
{"level":"info","ts":"2026-06-05T09:48:46.799Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"9fc57c91-cc49-4017-8bd1-b1b00fea8721","name":"stop-z5q6j-head"}
{"level":"info","ts":"2026-06-05T09:48:46.809Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"9fc57c91-cc49-4017-8bd1-b1b00fea8721","name":"stop-z5q6j-workers"}
{"level":"info","ts":"2026-06-05T09:48:46.809Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"9fc57c91-cc49-4017-8bd1-b1b00fea8721","cluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:48.780Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"9055775d-69ed-4e2d-8a31-20b52c805587"}
{"level":"info","ts":"2026-06-05T09:48:48.780Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"9055775d-69ed-4e2d-8a31-20b52c805587"}
{"level":"info","ts":"2026-06-05T09:48:48.780Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"9055775d-69ed-4e2d-8a31-20b52c805587","1 head service found":"stop-z5q6j-head-svc"}
{"level":"info","ts":"2026-06-05T09:48:48.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"9055775d-69ed-4e2d-8a31-20b52c805587","Found 1 head Pod":"stop-z5q6j-head-kpbk9","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:48:48.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"9055775d-69ed-4e2d-8a31-20b52c805587","head Pod":"stop-z5q6j-head-kpbk9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-z5q6j-head-kpbk9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:48:48.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"9055775d-69ed-4e2d-8a31-20b52c805587","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:48:48.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"9055775d-69ed-4e2d-8a31-20b52c805587","worker Pod":"stop-z5q6j-small-group-worker-kjlsx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-z5q6j-small-group-worker-kjlsx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:48:48.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"9055775d-69ed-4e2d-8a31-20b52c805587","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:48:48.781Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"9055775d-69ed-4e2d-8a31-20b52c805587","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:48:48.781Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"9055775d-69ed-4e2d-8a31-20b52c805587","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:48:48.781Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"9055775d-69ed-4e2d-8a31-20b52c805587","podName":"stop-z5q6j-small-group-worker-kjlsx","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:48:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-05T09:48:48.781Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"9055775d-69ed-4e2d-8a31-20b52c805587","podName":"stop-z5q6j-small-group-worker-kjlsx","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:48:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-05T09:48:48.781Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"9055775d-69ed-4e2d-8a31-20b52c805587","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:48:48.781Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"9055775d-69ed-4e2d-8a31-20b52c805587","seconds":300}
{"level":"info","ts":"2026-06-05T09:48:49.413Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"31195049-168f-4253-98bc-878a8008a24b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:48:49.413Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"31195049-168f-4253-98bc-878a8008a24b","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:49.413Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"31195049-168f-4253-98bc-878a8008a24b","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:48:49.413Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"31195049-168f-4253-98bc-878a8008a24b"}
{"level":"info","ts":"2026-06-05T09:48:49.413Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"31195049-168f-4253-98bc-878a8008a24b","RayCluster":"stop-z5q6j","State":""}
{"level":"info","ts":"2026-06-05T09:48:50.973Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"46edc0ee-2a10-4faa-853d-9729d5eb31de"}
{"level":"info","ts":"2026-06-05T09:48:50.973Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"46edc0ee-2a10-4faa-853d-9729d5eb31de"}
{"level":"info","ts":"2026-06-05T09:48:50.973Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"46edc0ee-2a10-4faa-853d-9729d5eb31de","1 head service found":"stop-z5q6j-head-svc"}
{"level":"info","ts":"2026-06-05T09:48:50.973Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"46edc0ee-2a10-4faa-853d-9729d5eb31de","Found 1 head Pod":"stop-z5q6j-head-kpbk9","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:48:50.973Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"46edc0ee-2a10-4faa-853d-9729d5eb31de","head Pod":"stop-z5q6j-head-kpbk9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-z5q6j-head-kpbk9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:48:50.973Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"46edc0ee-2a10-4faa-853d-9729d5eb31de","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:48:50.973Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"46edc0ee-2a10-4faa-853d-9729d5eb31de","worker Pod":"stop-z5q6j-small-group-worker-kjlsx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-z5q6j-small-group-worker-kjlsx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:48:50.973Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"46edc0ee-2a10-4faa-853d-9729d5eb31de","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:48:50.973Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"46edc0ee-2a10-4faa-853d-9729d5eb31de","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:48:50.974Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"46edc0ee-2a10-4faa-853d-9729d5eb31de","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:48:50.974Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"46edc0ee-2a10-4faa-853d-9729d5eb31de","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-06-05T09:48:50.974Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"46edc0ee-2a10-4faa-853d-9729d5eb31de","name":"stop-z5q6j","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:48:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.25","serviceIP":"10.132.0.25","podName":"stop-z5q6j-head-kpbk9","serviceName":"stop-z5q6j-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:48:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:48:50Z","stateTransitionTimes":{"ready":"2026-06-05T09:48:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.25","serviceIP":"10.132.0.25","podName":"stop-z5q6j-head-kpbk9","serviceName":"stop-z5q6j-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:48:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:48:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:48:50.987Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"ff48aedb-9ee2-4961-b827-3239c30595a4","cluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:50.987Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"b798438c-009a-47ab-b741-483920905a8f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:48:50.987Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"b798438c-009a-47ab-b741-483920905a8f","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:50.987Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"b798438c-009a-47ab-b741-483920905a8f","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:48:50.987Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"30889cb7-0e78-4529-987c-f7b78d23bf58","rayCluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:50.987Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"30889cb7-0e78-4529-987c-f7b78d23bf58","pod":"stop-z5q6j-head-kpbk9","ip":"10.132.0.25"}
{"level":"info","ts":"2026-06-05T09:48:50.987Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"30889cb7-0e78-4529-987c-f7b78d23bf58","pod":"stop-z5q6j-small-group-worker-kjlsx","ip":"10.133.0.51"}
{"level":"info","ts":"2026-06-05T09:48:50.987Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"b798438c-009a-47ab-b741-483920905a8f"}
{"level":"info","ts":"2026-06-05T09:48:50.987Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"b798438c-009a-47ab-b741-483920905a8f","head service name":"stop-z5q6j-head-svc","namespace":"test-ns-thrrw"}
{"level":"info","ts":"2026-06-05T09:48:50.987Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"b798438c-009a-47ab-b741-483920905a8f","head service URL":"stop-z5q6j-head-svc.test-ns-thrrw.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-05T09:48:50.987Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"30889cb7-0e78-4529-987c-f7b78d23bf58","rayCluster":"stop-z5q6j","podIPs":["10.132.0.25","10.133.0.51"]}
{"level":"info","ts":"2026-06-05T09:48:50.987Z","logger":"controllers.RayJob","msg":"user-provided submitter template is used; the first container is assumed to be the submitter","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"b798438c-009a-47ab-b741-483920905a8f"}
{"level":"info","ts":"2026-06-05T09:48:50.987Z","logger":"controllers.RayJob","msg":"No command is specified in the user-provided template. Default command is used","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"b798438c-009a-47ab-b741-483920905a8f","command":["if","!","ray","job","status","--address","http://stop-z5q6j-head-svc.test-ns-thrrw.svc.cluster.local:8265","stop-rnf5t",">/dev/null","2>&1",";","then","ray","job","submit","--address","http://stop-z5q6j-head-svc.test-ns-thrrw.svc.cluster.local:8265","--no-wait","--submission-id","stop-rnf5t","--","python /home/ray/jobs/stop.py",";","fi",";","ray","job","logs","--address","http://stop-z5q6j-head-svc.test-ns-thrrw.svc.cluster.local:8265","--follow","stop-rnf5t"]}
{"level":"info","ts":"2026-06-05T09:48:50.996Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"b798438c-009a-47ab-b741-483920905a8f","Kubernetes Job":"stop"}
{"level":"info","ts":"2026-06-05T09:48:50.996Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"b798438c-009a-47ab-b741-483920905a8f","SubmissionMode":"K8sJobMode","RayCluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:50.996Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"b798438c-009a-47ab-b741-483920905a8f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-rnf5t","rayClusterName":"stop-z5q6j","jobDeploymentStatus":"Initializing","startTime":"2026-06-05T09:48:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-rnf5t","rayClusterName":"stop-z5q6j","dashboardURL":"stop-z5q6j-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-05T09:48:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:48:50.996Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"b798438c-009a-47ab-b741-483920905a8f","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-05T09:48:51.000Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"ff48aedb-9ee2-4961-b827-3239c30595a4","name":"stop-z5q6j-head"}
{"level":"info","ts":"2026-06-05T09:48:51.008Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"1d785f39-2dee-4824-8bbc-a550b41d2f2b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:48:51.008Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"1d785f39-2dee-4824-8bbc-a550b41d2f2b","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:51.008Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"1d785f39-2dee-4824-8bbc-a550b41d2f2b","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:48:51.008Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"1d785f39-2dee-4824-8bbc-a550b41d2f2b"}
{"level":"info","ts":"2026-06-05T09:48:51.008Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"1d785f39-2dee-4824-8bbc-a550b41d2f2b","head service name":"stop-z5q6j-head-svc","namespace":"test-ns-thrrw"}
{"level":"info","ts":"2026-06-05T09:48:51.008Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"1d785f39-2dee-4824-8bbc-a550b41d2f2b","head service URL":"stop-z5q6j-head-svc.test-ns-thrrw.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-05T09:48:51.008Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job for RayJob already exists","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"1d785f39-2dee-4824-8bbc-a550b41d2f2b","Kubernetes Job":"stop"}
{"level":"info","ts":"2026-06-05T09:48:51.008Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"1d785f39-2dee-4824-8bbc-a550b41d2f2b","SubmissionMode":"K8sJobMode","RayCluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:51.008Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"1d785f39-2dee-4824-8bbc-a550b41d2f2b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-rnf5t","rayClusterName":"stop-z5q6j","jobDeploymentStatus":"Initializing","startTime":"2026-06-05T09:48:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-rnf5t","rayClusterName":"stop-z5q6j","dashboardURL":"stop-z5q6j-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-05T09:48:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:48:51.008Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"1d785f39-2dee-4824-8bbc-a550b41d2f2b","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-05T09:48:51.016Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"ff48aedb-9ee2-4961-b827-3239c30595a4","name":"stop-z5q6j-workers"}
{"level":"info","ts":"2026-06-05T09:48:51.016Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"ff48aedb-9ee2-4961-b827-3239c30595a4","cluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:51.020Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"1d785f39-2dee-4824-8bbc-a550b41d2f2b","error":"Operation cannot be fulfilled on rayjobs.ray.io \"stop\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-06-05T09:48:51.020Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"1d785f39-2dee-4824-8bbc-a550b41d2f2b"}
{"level":"error","ts":"2026-06-05T09:48:51.020Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"1d785f39-2dee-4824-8bbc-a550b41d2f2b","error":"Operation cannot be fulfilled on rayjobs.ray.io \"stop\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:51.020Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"189284f1-22b7-4064-b409-db3fd7fe3c0d","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:48:51.020Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"189284f1-22b7-4064-b409-db3fd7fe3c0d","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:51.020Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"189284f1-22b7-4064-b409-db3fd7fe3c0d","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:48:51.020Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"189284f1-22b7-4064-b409-db3fd7fe3c0d"}
{"level":"error","ts":"2026-06-05T09:48:51.029Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"189284f1-22b7-4064-b409-db3fd7fe3c0d","JobId":"stop-rnf5t","error":"Job stop-rnf5t does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:51.029Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"189284f1-22b7-4064-b409-db3fd7fe3c0d"}
{"level":"error","ts":"2026-06-05T09:48:51.029Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"189284f1-22b7-4064-b409-db3fd7fe3c0d","error":"Job stop-rnf5t does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:51.030Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"2d6e64b8-f105-4f9f-86aa-8bb20fb44773","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:48:51.030Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"2d6e64b8-f105-4f9f-86aa-8bb20fb44773","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:51.030Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"2d6e64b8-f105-4f9f-86aa-8bb20fb44773","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:48:51.030Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"2d6e64b8-f105-4f9f-86aa-8bb20fb44773"}
{"level":"error","ts":"2026-06-05T09:48:51.036Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"2d6e64b8-f105-4f9f-86aa-8bb20fb44773","JobId":"stop-rnf5t","error":"Job stop-rnf5t does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:51.036Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"2d6e64b8-f105-4f9f-86aa-8bb20fb44773"}
{"level":"error","ts":"2026-06-05T09:48:51.036Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"2d6e64b8-f105-4f9f-86aa-8bb20fb44773","error":"Job stop-rnf5t does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:51.040Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"d4f08af1-3d8a-4785-b224-ef44a44d0c1f","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:48:51.040Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"d4f08af1-3d8a-4785-b224-ef44a44d0c1f","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:51.040Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"d4f08af1-3d8a-4785-b224-ef44a44d0c1f","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:48:51.040Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"d4f08af1-3d8a-4785-b224-ef44a44d0c1f"}
{"level":"error","ts":"2026-06-05T09:48:51.110Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"d4f08af1-3d8a-4785-b224-ef44a44d0c1f","JobId":"stop-rnf5t","error":"Job stop-rnf5t does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:51.110Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"d4f08af1-3d8a-4785-b224-ef44a44d0c1f"}
{"level":"error","ts":"2026-06-05T09:48:51.110Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"d4f08af1-3d8a-4785-b224-ef44a44d0c1f","error":"Job stop-rnf5t does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:51.150Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"d76cc960-fc84-4520-b6f3-5e8ce0c8fc23","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:48:51.150Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"d76cc960-fc84-4520-b6f3-5e8ce0c8fc23","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:51.150Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"d76cc960-fc84-4520-b6f3-5e8ce0c8fc23","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:48:51.150Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"d76cc960-fc84-4520-b6f3-5e8ce0c8fc23"}
{"level":"error","ts":"2026-06-05T09:48:51.155Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"d76cc960-fc84-4520-b6f3-5e8ce0c8fc23","JobId":"stop-rnf5t","error":"Job stop-rnf5t does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:51.156Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"d76cc960-fc84-4520-b6f3-5e8ce0c8fc23"}
{"level":"error","ts":"2026-06-05T09:48:51.156Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"d76cc960-fc84-4520-b6f3-5e8ce0c8fc23","error":"Job stop-rnf5t does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:51.236Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"73f00fd3-681d-4b8b-b448-bd4ba6e5d077","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:48:51.236Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"73f00fd3-681d-4b8b-b448-bd4ba6e5d077","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:51.236Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"73f00fd3-681d-4b8b-b448-bd4ba6e5d077","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:48:51.236Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"73f00fd3-681d-4b8b-b448-bd4ba6e5d077"}
{"level":"error","ts":"2026-06-05T09:48:51.241Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"73f00fd3-681d-4b8b-b448-bd4ba6e5d077","JobId":"stop-rnf5t","error":"Job stop-rnf5t does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:51.241Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"73f00fd3-681d-4b8b-b448-bd4ba6e5d077"}
{"level":"error","ts":"2026-06-05T09:48:51.241Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"73f00fd3-681d-4b8b-b448-bd4ba6e5d077","error":"Job stop-rnf5t does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:51.402Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"c0729952-ccf8-4f4f-b647-ae91d5d0cdba","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:48:51.402Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"c0729952-ccf8-4f4f-b647-ae91d5d0cdba","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:51.402Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"c0729952-ccf8-4f4f-b647-ae91d5d0cdba","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:48:51.402Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"c0729952-ccf8-4f4f-b647-ae91d5d0cdba"}
{"level":"error","ts":"2026-06-05T09:48:51.408Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"c0729952-ccf8-4f4f-b647-ae91d5d0cdba","JobId":"stop-rnf5t","error":"Job stop-rnf5t does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:51.408Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"c0729952-ccf8-4f4f-b647-ae91d5d0cdba"}
{"level":"error","ts":"2026-06-05T09:48:51.408Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"c0729952-ccf8-4f4f-b647-ae91d5d0cdba","error":"Job stop-rnf5t does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:51.728Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"f027264b-690c-4962-b904-37fffd10f05f","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:48:51.728Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"f027264b-690c-4962-b904-37fffd10f05f","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:51.728Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"f027264b-690c-4962-b904-37fffd10f05f","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:48:51.729Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"f027264b-690c-4962-b904-37fffd10f05f"}
{"level":"error","ts":"2026-06-05T09:48:51.734Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"f027264b-690c-4962-b904-37fffd10f05f","JobId":"stop-rnf5t","error":"Job stop-rnf5t does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:51.734Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"f027264b-690c-4962-b904-37fffd10f05f"}
{"level":"error","ts":"2026-06-05T09:48:51.734Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"f027264b-690c-4962-b904-37fffd10f05f","error":"Job stop-rnf5t does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:52.019Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"636f5b6f-97b0-428d-9976-e5dcac111b85","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:48:52.019Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"636f5b6f-97b0-428d-9976-e5dcac111b85","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:52.019Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"636f5b6f-97b0-428d-9976-e5dcac111b85","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:48:52.019Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"636f5b6f-97b0-428d-9976-e5dcac111b85"}
{"level":"error","ts":"2026-06-05T09:48:52.025Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"636f5b6f-97b0-428d-9976-e5dcac111b85","JobId":"stop-rnf5t","error":"Job stop-rnf5t does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:52.025Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"636f5b6f-97b0-428d-9976-e5dcac111b85"}
{"level":"error","ts":"2026-06-05T09:48:52.025Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"636f5b6f-97b0-428d-9976-e5dcac111b85","error":"Job stop-rnf5t does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:52.375Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"c6f0bbaa-c5a9-4c67-8f83-28c145a34403","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:48:52.375Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"c6f0bbaa-c5a9-4c67-8f83-28c145a34403","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:52.375Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"c6f0bbaa-c5a9-4c67-8f83-28c145a34403","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:48:52.376Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"c6f0bbaa-c5a9-4c67-8f83-28c145a34403"}
{"level":"error","ts":"2026-06-05T09:48:52.381Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"c6f0bbaa-c5a9-4c67-8f83-28c145a34403","JobId":"stop-rnf5t","error":"Job stop-rnf5t does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:52.381Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"c6f0bbaa-c5a9-4c67-8f83-28c145a34403"}
{"level":"error","ts":"2026-06-05T09:48:52.381Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"c6f0bbaa-c5a9-4c67-8f83-28c145a34403","error":"Job stop-rnf5t does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:52.988Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"f2e0c256-af39-4e08-8825-0ddb2722a484"}
{"level":"info","ts":"2026-06-05T09:48:52.988Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"f2e0c256-af39-4e08-8825-0ddb2722a484"}
{"level":"info","ts":"2026-06-05T09:48:52.988Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"f2e0c256-af39-4e08-8825-0ddb2722a484","1 head service found":"stop-z5q6j-head-svc"}
{"level":"info","ts":"2026-06-05T09:48:52.988Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"f2e0c256-af39-4e08-8825-0ddb2722a484","Found 1 head Pod":"stop-z5q6j-head-kpbk9","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:48:52.988Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"f2e0c256-af39-4e08-8825-0ddb2722a484","head Pod":"stop-z5q6j-head-kpbk9","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-z5q6j-head-kpbk9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:48:52.988Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"f2e0c256-af39-4e08-8825-0ddb2722a484","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:48:52.988Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"f2e0c256-af39-4e08-8825-0ddb2722a484","worker Pod":"stop-z5q6j-small-group-worker-kjlsx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-z5q6j-small-group-worker-kjlsx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:48:52.988Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"f2e0c256-af39-4e08-8825-0ddb2722a484","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:48:52.988Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"f2e0c256-af39-4e08-8825-0ddb2722a484","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:48:52.988Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"f2e0c256-af39-4e08-8825-0ddb2722a484","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:48:52.988Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"f2e0c256-af39-4e08-8825-0ddb2722a484","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:48:52.988Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"f2e0c256-af39-4e08-8825-0ddb2722a484","seconds":300}
{"level":"info","ts":"2026-06-05T09:48:54.942Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"1c33cd75-643e-4dee-a972-8b6a3a56d1ca","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:48:54.942Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"1c33cd75-643e-4dee-a972-8b6a3a56d1ca","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:54.942Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"1c33cd75-643e-4dee-a972-8b6a3a56d1ca","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:48:54.942Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"1c33cd75-643e-4dee-a972-8b6a3a56d1ca"}
{"level":"error","ts":"2026-06-05T09:48:54.947Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"1c33cd75-643e-4dee-a972-8b6a3a56d1ca","JobId":"stop-rnf5t","error":"Job stop-rnf5t does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:54.947Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"1c33cd75-643e-4dee-a972-8b6a3a56d1ca"}
{"level":"error","ts":"2026-06-05T09:48:54.947Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"1c33cd75-643e-4dee-a972-8b6a3a56d1ca","error":"Job stop-rnf5t does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:48:55.623Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"2921f1e5-46fd-469b-be64-191c396b5edc","rayCluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:48:55.623Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"2921f1e5-46fd-469b-be64-191c396b5edc","pod":"stop-z5q6j-head-kpbk9","ip":"10.132.0.25"}
{"level":"info","ts":"2026-06-05T09:48:55.623Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"2921f1e5-46fd-469b-be64-191c396b5edc","pod":"stop-z5q6j-small-group-worker-kjlsx","ip":"10.133.0.51"}
{"level":"info","ts":"2026-06-05T09:48:55.623Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"2921f1e5-46fd-469b-be64-191c396b5edc","rayCluster":"stop-z5q6j","podIPs":["10.132.0.25","10.133.0.51"]}
{"level":"info","ts":"2026-06-05T09:49:00.068Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"46a7c68d-4b74-4125-8485-ecbbb82c8008","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:00.068Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"46a7c68d-4b74-4125-8485-ecbbb82c8008","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:49:00.068Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"46a7c68d-4b74-4125-8485-ecbbb82c8008","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:49:00.069Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"46a7c68d-4b74-4125-8485-ecbbb82c8008"}
{"level":"info","ts":"2026-06-05T09:49:00.075Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"46a7c68d-4b74-4125-8485-ecbbb82c8008","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-rnf5t","rayClusterName":"stop-z5q6j","dashboardURL":"stop-z5q6j-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-05T09:48:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:48:58Z"},"jobId":"stop-rnf5t","rayClusterName":"stop-z5q6j","dashboardURL":"stop-z5q6j-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-05T09:48:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:48:50Z","stateTransitionTimes":{"ready":"2026-06-05T09:48:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.25","serviceIP":"10.132.0.25","podName":"stop-z5q6j-head-kpbk9","serviceName":"stop-z5q6j-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:48:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:48:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:49:00.075Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"46a7c68d-4b74-4125-8485-ecbbb82c8008","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-05T09:49:00.092Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"9a07a806-20a7-4640-bf22-f893b9b5b0f5","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:00.092Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"9a07a806-20a7-4640-bf22-f893b9b5b0f5","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:49:00.092Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"9a07a806-20a7-4640-bf22-f893b9b5b0f5","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:49:00.093Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"9a07a806-20a7-4640-bf22-f893b9b5b0f5"}
{"level":"info","ts":"2026-06-05T09:49:00.106Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"9a07a806-20a7-4640-bf22-f893b9b5b0f5","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:48:58Z"},"jobId":"stop-rnf5t","rayClusterName":"stop-z5q6j","dashboardURL":"stop-z5q6j-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-05T09:48:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:48:50Z","stateTransitionTimes":{"ready":"2026-06-05T09:48:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.25","serviceIP":"10.132.0.25","podName":"stop-z5q6j-head-kpbk9","serviceName":"stop-z5q6j-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:48:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:48:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:48:58Z"},"jobId":"stop-rnf5t","rayClusterName":"stop-z5q6j","dashboardURL":"stop-z5q6j-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-05T09:48:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:48:50Z","stateTransitionTimes":{"ready":"2026-06-05T09:48:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.25","serviceIP":"10.132.0.25","podName":"stop-z5q6j-head-kpbk9","serviceName":"stop-z5q6j-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:48:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:48:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:49:03.092Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"af0b1b6c-e5be-4819-a825-9e363f8cdbd4","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:03.092Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"af0b1b6c-e5be-4819-a825-9e363f8cdbd4","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:49:03.092Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"af0b1b6c-e5be-4819-a825-9e363f8cdbd4","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:49:03.092Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"af0b1b6c-e5be-4819-a825-9e363f8cdbd4"}
{"level":"info","ts":"2026-06-05T09:49:03.095Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"af0b1b6c-e5be-4819-a825-9e363f8cdbd4","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:48:58Z"},"jobId":"stop-rnf5t","rayClusterName":"stop-z5q6j","dashboardURL":"stop-z5q6j-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-05T09:48:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:48:50Z","stateTransitionTimes":{"ready":"2026-06-05T09:48:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.25","serviceIP":"10.132.0.25","podName":"stop-z5q6j-head-kpbk9","serviceName":"stop-z5q6j-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:48:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:48:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:48:58Z"},"jobId":"stop-rnf5t","rayClusterName":"stop-z5q6j","dashboardURL":"stop-z5q6j-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-05T09:48:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:48:50Z","stateTransitionTimes":{"ready":"2026-06-05T09:48:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.25","serviceIP":"10.132.0.25","podName":"stop-z5q6j-head-kpbk9","serviceName":"stop-z5q6j-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:48:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:48:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:49:03.095Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"af0b1b6c-e5be-4819-a825-9e363f8cdbd4","old JobStatus":"PENDING","new JobStatus":"RUNNING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-05T09:49:03.109Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"e1386670-3ef3-4855-852a-3742c56c8b47","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:03.109Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"e1386670-3ef3-4855-852a-3742c56c8b47","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:49:03.109Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"e1386670-3ef3-4855-852a-3742c56c8b47","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:49:03.109Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"e1386670-3ef3-4855-852a-3742c56c8b47"}
{"level":"info","ts":"2026-06-05T09:49:03.111Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"e1386670-3ef3-4855-852a-3742c56c8b47","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:48:58Z"},"jobId":"stop-rnf5t","rayClusterName":"stop-z5q6j","dashboardURL":"stop-z5q6j-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-05T09:48:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:48:50Z","stateTransitionTimes":{"ready":"2026-06-05T09:48:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.25","serviceIP":"10.132.0.25","podName":"stop-z5q6j-head-kpbk9","serviceName":"stop-z5q6j-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:48:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:48:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:48:58Z"},"jobId":"stop-rnf5t","rayClusterName":"stop-z5q6j","dashboardURL":"stop-z5q6j-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-05T09:48:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:48:50Z","stateTransitionTimes":{"ready":"2026-06-05T09:48:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.25","serviceIP":"10.132.0.25","podName":"stop-z5q6j-head-kpbk9","serviceName":"stop-z5q6j-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:48:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:48:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:49:04.001Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"594e9d12-a2f8-4c33-bc14-ea8ba8e9cdd7"}
{"level":"info","ts":"2026-06-05T09:49:04.001Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"594e9d12-a2f8-4c33-bc14-ea8ba8e9cdd7"}
{"level":"info","ts":"2026-06-05T09:49:04.001Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"594e9d12-a2f8-4c33-bc14-ea8ba8e9cdd7","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-05T09:49:04.001Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"594e9d12-a2f8-4c33-bc14-ea8ba8e9cdd7","Found 1 head Pod":"raycluster-head-vjd7d","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:49:04.001Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"594e9d12-a2f8-4c33-bc14-ea8ba8e9cdd7","head Pod":"raycluster-head-vjd7d","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-vjd7d. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:49:04.001Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"594e9d12-a2f8-4c33-bc14-ea8ba8e9cdd7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:49:04.001Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"594e9d12-a2f8-4c33-bc14-ea8ba8e9cdd7","worker Pod":"raycluster-small-group-worker-j9pbw","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-j9pbw. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:49:04.001Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"594e9d12-a2f8-4c33-bc14-ea8ba8e9cdd7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:04.001Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"594e9d12-a2f8-4c33-bc14-ea8ba8e9cdd7","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:49:04.001Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"594e9d12-a2f8-4c33-bc14-ea8ba8e9cdd7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:04.001Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"594e9d12-a2f8-4c33-bc14-ea8ba8e9cdd7","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:49:04.001Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"reconcileID":"594e9d12-a2f8-4c33-bc14-ea8ba8e9cdd7","seconds":300}
{"level":"info","ts":"2026-06-05T09:49:06.109Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"df7b50d1-8e1e-43a2-8047-a37757dbdb9f","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:06.109Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"df7b50d1-8e1e-43a2-8047-a37757dbdb9f","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:49:06.109Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"df7b50d1-8e1e-43a2-8047-a37757dbdb9f","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:49:06.109Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"df7b50d1-8e1e-43a2-8047-a37757dbdb9f"}
{"level":"info","ts":"2026-06-05T09:49:06.112Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"df7b50d1-8e1e-43a2-8047-a37757dbdb9f","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:48:58Z"},"jobId":"stop-rnf5t","rayClusterName":"stop-z5q6j","dashboardURL":"stop-z5q6j-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-05T09:48:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:48:50Z","stateTransitionTimes":{"ready":"2026-06-05T09:48:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.25","serviceIP":"10.132.0.25","podName":"stop-z5q6j-head-kpbk9","serviceName":"stop-z5q6j-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:48:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:48:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:48:58Z"},"jobId":"stop-rnf5t","rayClusterName":"stop-z5q6j","dashboardURL":"stop-z5q6j-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-05T09:48:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:48:50Z","stateTransitionTimes":{"ready":"2026-06-05T09:48:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.25","serviceIP":"10.132.0.25","podName":"stop-z5q6j-head-kpbk9","serviceName":"stop-z5q6j-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:48:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:48:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:49:09.113Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"d401c16e-ecf3-40c0-884b-dc6f3b2e4c7a","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:09.113Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"d401c16e-ecf3-40c0-884b-dc6f3b2e4c7a","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:49:09.113Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"d401c16e-ecf3-40c0-884b-dc6f3b2e4c7a","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:49:09.113Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"d401c16e-ecf3-40c0-884b-dc6f3b2e4c7a"}
{"level":"info","ts":"2026-06-05T09:49:09.116Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"d401c16e-ecf3-40c0-884b-dc6f3b2e4c7a","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:48:58Z"},"jobId":"stop-rnf5t","rayClusterName":"stop-z5q6j","dashboardURL":"stop-z5q6j-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-05T09:48:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:48:50Z","stateTransitionTimes":{"ready":"2026-06-05T09:48:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.25","serviceIP":"10.132.0.25","podName":"stop-z5q6j-head-kpbk9","serviceName":"stop-z5q6j-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:48:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:48:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:48:58Z"},"jobId":"stop-rnf5t","rayClusterName":"stop-z5q6j","dashboardURL":"stop-z5q6j-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-05T09:48:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:48:50Z","stateTransitionTimes":{"ready":"2026-06-05T09:48:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.25","serviceIP":"10.132.0.25","podName":"stop-z5q6j-head-kpbk9","serviceName":"stop-z5q6j-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:48:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:48:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:49:12.117Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"5075f521-bb68-4b3f-9a6f-1ed9cfdebdfd","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:12.117Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"5075f521-bb68-4b3f-9a6f-1ed9cfdebdfd","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:49:12.117Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"5075f521-bb68-4b3f-9a6f-1ed9cfdebdfd","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:49:12.118Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"5075f521-bb68-4b3f-9a6f-1ed9cfdebdfd"}
{"level":"info","ts":"2026-06-05T09:49:12.121Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"5075f521-bb68-4b3f-9a6f-1ed9cfdebdfd","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:48:58Z"},"jobId":"stop-rnf5t","rayClusterName":"stop-z5q6j","dashboardURL":"stop-z5q6j-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-05T09:48:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:48:50Z","stateTransitionTimes":{"ready":"2026-06-05T09:48:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.25","serviceIP":"10.132.0.25","podName":"stop-z5q6j-head-kpbk9","serviceName":"stop-z5q6j-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:48:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:48:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:48:58Z"},"jobId":"stop-rnf5t","rayClusterName":"stop-z5q6j","dashboardURL":"stop-z5q6j-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-05T09:48:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:48:50Z","stateTransitionTimes":{"ready":"2026-06-05T09:48:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.25","serviceIP":"10.132.0.25","podName":"stop-z5q6j-head-kpbk9","serviceName":"stop-z5q6j-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:48:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:48:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:49:15.122Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"1367b859-faba-4876-997b-81ef13a0a4bc","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:15.122Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"1367b859-faba-4876-997b-81ef13a0a4bc","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:49:15.122Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"1367b859-faba-4876-997b-81ef13a0a4bc","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:49:15.122Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"1367b859-faba-4876-997b-81ef13a0a4bc"}
{"level":"info","ts":"2026-06-05T09:49:15.125Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"1367b859-faba-4876-997b-81ef13a0a4bc","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:48:58Z"},"jobId":"stop-rnf5t","rayClusterName":"stop-z5q6j","dashboardURL":"stop-z5q6j-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-05T09:48:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:48:50Z","stateTransitionTimes":{"ready":"2026-06-05T09:48:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.25","serviceIP":"10.132.0.25","podName":"stop-z5q6j-head-kpbk9","serviceName":"stop-z5q6j-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:48:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:48:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:48:58Z"},"jobId":"stop-rnf5t","rayClusterName":"stop-z5q6j","dashboardURL":"stop-z5q6j-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-05T09:48:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:48:50Z","stateTransitionTimes":{"ready":"2026-06-05T09:48:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.25","serviceIP":"10.132.0.25","podName":"stop-z5q6j-head-kpbk9","serviceName":"stop-z5q6j-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:48:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:48:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:49:18.126Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"021fce00-95a6-4da0-8bfe-1a5f788a2770","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:18.126Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"021fce00-95a6-4da0-8bfe-1a5f788a2770","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:49:18.126Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"021fce00-95a6-4da0-8bfe-1a5f788a2770","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:49:18.126Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"021fce00-95a6-4da0-8bfe-1a5f788a2770"}
{"level":"info","ts":"2026-06-05T09:49:18.129Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"021fce00-95a6-4da0-8bfe-1a5f788a2770","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:48:58Z"},"jobId":"stop-rnf5t","rayClusterName":"stop-z5q6j","dashboardURL":"stop-z5q6j-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-05T09:48:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:48:50Z","stateTransitionTimes":{"ready":"2026-06-05T09:48:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.25","serviceIP":"10.132.0.25","podName":"stop-z5q6j-head-kpbk9","serviceName":"stop-z5q6j-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:48:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:48:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:48:58Z"},"jobId":"stop-rnf5t","rayClusterName":"stop-z5q6j","dashboardURL":"stop-z5q6j-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-05T09:48:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:48:50Z","stateTransitionTimes":{"ready":"2026-06-05T09:48:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.25","serviceIP":"10.132.0.25","podName":"stop-z5q6j-head-kpbk9","serviceName":"stop-z5q6j-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:48:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:48:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:49:21.129Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"11a0d367-a61b-4ef2-868f-97c6aa118b24","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:21.129Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"11a0d367-a61b-4ef2-868f-97c6aa118b24","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:49:21.129Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"11a0d367-a61b-4ef2-868f-97c6aa118b24","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:49:21.129Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"11a0d367-a61b-4ef2-868f-97c6aa118b24"}
{"level":"info","ts":"2026-06-05T09:49:21.132Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"11a0d367-a61b-4ef2-868f-97c6aa118b24","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:48:58Z"},"jobId":"stop-rnf5t","rayClusterName":"stop-z5q6j","dashboardURL":"stop-z5q6j-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-05T09:48:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:48:50Z","stateTransitionTimes":{"ready":"2026-06-05T09:48:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.25","serviceIP":"10.132.0.25","podName":"stop-z5q6j-head-kpbk9","serviceName":"stop-z5q6j-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:48:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:48:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:48:58Z"},"jobId":"stop-rnf5t","rayClusterName":"stop-z5q6j","dashboardURL":"stop-z5q6j-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-05T09:48:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:48:50Z","stateTransitionTimes":{"ready":"2026-06-05T09:48:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.25","serviceIP":"10.132.0.25","podName":"stop-z5q6j-head-kpbk9","serviceName":"stop-z5q6j-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:48:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:48:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:49:24.133Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"49ced82e-1dcb-4a11-96e5-d317506da88a","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:24.133Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"49ced82e-1dcb-4a11-96e5-d317506da88a","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:49:24.133Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"49ced82e-1dcb-4a11-96e5-d317506da88a","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:49:24.133Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"49ced82e-1dcb-4a11-96e5-d317506da88a"}
{"level":"info","ts":"2026-06-05T09:49:24.136Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"49ced82e-1dcb-4a11-96e5-d317506da88a","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:48:58Z"},"jobId":"stop-rnf5t","rayClusterName":"stop-z5q6j","dashboardURL":"stop-z5q6j-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-05T09:48:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:48:50Z","stateTransitionTimes":{"ready":"2026-06-05T09:48:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.25","serviceIP":"10.132.0.25","podName":"stop-z5q6j-head-kpbk9","serviceName":"stop-z5q6j-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:48:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:48:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:48:58Z","endTime":"2026-06-05T09:49:21Z"},"jobId":"stop-rnf5t","rayClusterName":"stop-z5q6j","dashboardURL":"stop-z5q6j-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-06-05T09:48:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:48:50Z","stateTransitionTimes":{"ready":"2026-06-05T09:48:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.25","serviceIP":"10.132.0.25","podName":"stop-z5q6j-head-kpbk9","serviceName":"stop-z5q6j-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:48:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:48:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:49:24.136Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"49ced82e-1dcb-4a11-96e5-d317506da88a","old JobStatus":"RUNNING","new JobStatus":"STOPPED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-05T09:49:24.153Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"5adead55-f465-4574-abea-479841f3676d","JobStatus":"STOPPED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:24.153Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"5adead55-f465-4574-abea-479841f3676d","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:49:24.153Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"5adead55-f465-4574-abea-479841f3676d","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:49:24.153Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"5adead55-f465-4574-abea-479841f3676d"}
{"level":"info","ts":"2026-06-05T09:49:24.155Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"5adead55-f465-4574-abea-479841f3676d","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:48:58Z","endTime":"2026-06-05T09:49:21Z"},"jobId":"stop-rnf5t","rayClusterName":"stop-z5q6j","dashboardURL":"stop-z5q6j-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-06-05T09:48:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:48:50Z","stateTransitionTimes":{"ready":"2026-06-05T09:48:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.25","serviceIP":"10.132.0.25","podName":"stop-z5q6j-head-kpbk9","serviceName":"stop-z5q6j-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:48:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:48:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:48:58Z","endTime":"2026-06-05T09:49:21Z"},"jobId":"stop-rnf5t","rayClusterName":"stop-z5q6j","dashboardURL":"stop-z5q6j-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-06-05T09:48:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:48:50Z","stateTransitionTimes":{"ready":"2026-06-05T09:48:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.25","serviceIP":"10.132.0.25","podName":"stop-z5q6j-head-kpbk9","serviceName":"stop-z5q6j-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:48:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:48:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:49:27.154Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"62a607d1-3627-4ca5-906b-86256db2c803","JobStatus":"STOPPED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:27.154Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"62a607d1-3627-4ca5-906b-86256db2c803","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:49:27.154Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"62a607d1-3627-4ca5-906b-86256db2c803","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:49:27.154Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"62a607d1-3627-4ca5-906b-86256db2c803"}
{"level":"info","ts":"2026-06-05T09:49:27.157Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"62a607d1-3627-4ca5-906b-86256db2c803","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:48:58Z","endTime":"2026-06-05T09:49:21Z"},"jobId":"stop-rnf5t","rayClusterName":"stop-z5q6j","dashboardURL":"stop-z5q6j-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-06-05T09:48:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:48:50Z","stateTransitionTimes":{"ready":"2026-06-05T09:48:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.25","serviceIP":"10.132.0.25","podName":"stop-z5q6j-head-kpbk9","serviceName":"stop-z5q6j-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:48:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:48:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:48:58Z","endTime":"2026-06-05T09:49:21Z"},"jobId":"stop-rnf5t","rayClusterName":"stop-z5q6j","dashboardURL":"stop-z5q6j-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-06-05T09:48:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:48:50Z","stateTransitionTimes":{"ready":"2026-06-05T09:48:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.25","serviceIP":"10.132.0.25","podName":"stop-z5q6j-head-kpbk9","serviceName":"stop-z5q6j-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:48:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:48:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:49:27.208Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"ae4adc96-95fc-4861-8b3c-6defab271b14","JobStatus":"STOPPED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:27.208Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"ae4adc96-95fc-4861-8b3c-6defab271b14","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:49:27.208Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"ae4adc96-95fc-4861-8b3c-6defab271b14","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:49:27.208Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"ae4adc96-95fc-4861-8b3c-6defab271b14"}
{"level":"info","ts":"2026-06-05T09:49:27.211Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"ae4adc96-95fc-4861-8b3c-6defab271b14","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:48:58Z","endTime":"2026-06-05T09:49:21Z"},"jobId":"stop-rnf5t","rayClusterName":"stop-z5q6j","dashboardURL":"stop-z5q6j-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-06-05T09:48:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:48:50Z","stateTransitionTimes":{"ready":"2026-06-05T09:48:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.25","serviceIP":"10.132.0.25","podName":"stop-z5q6j-head-kpbk9","serviceName":"stop-z5q6j-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:48:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:48:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:48:58Z","endTime":"2026-06-05T09:49:21Z"},"jobId":"stop-rnf5t","rayClusterName":"stop-z5q6j","dashboardURL":"stop-z5q6j-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-06-05T09:48:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:48:50Z","stateTransitionTimes":{"ready":"2026-06-05T09:48:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.25","serviceIP":"10.132.0.25","podName":"stop-z5q6j-head-kpbk9","serviceName":"stop-z5q6j-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:48:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:48:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:49:28.226Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"cb3ed495-c460-4974-8fcd-efb79a5acc90","JobStatus":"STOPPED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:28.226Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"cb3ed495-c460-4974-8fcd-efb79a5acc90","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:49:28.227Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"cb3ed495-c460-4974-8fcd-efb79a5acc90","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:49:28.227Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"cb3ed495-c460-4974-8fcd-efb79a5acc90"}
{"level":"info","ts":"2026-06-05T09:49:28.230Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"cb3ed495-c460-4974-8fcd-efb79a5acc90","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:48:58Z","endTime":"2026-06-05T09:49:21Z"},"jobId":"stop-rnf5t","rayClusterName":"stop-z5q6j","dashboardURL":"stop-z5q6j-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-06-05T09:48:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:48:50Z","stateTransitionTimes":{"ready":"2026-06-05T09:48:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.25","serviceIP":"10.132.0.25","podName":"stop-z5q6j-head-kpbk9","serviceName":"stop-z5q6j-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:48:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:48:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:48:58Z","endTime":"2026-06-05T09:49:21Z"},"jobId":"stop-rnf5t","rayClusterName":"stop-z5q6j","dashboardURL":"stop-z5q6j-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-06-05T09:48:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:48:50Z","stateTransitionTimes":{"ready":"2026-06-05T09:48:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.25","serviceIP":"10.132.0.25","podName":"stop-z5q6j-head-kpbk9","serviceName":"stop-z5q6j-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:48:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:48:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:49:28.259Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"fc1395d3-54d9-40e7-a5db-ac54fb743e43","JobStatus":"STOPPED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:28.259Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"fc1395d3-54d9-40e7-a5db-ac54fb743e43","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:49:28.259Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"fc1395d3-54d9-40e7-a5db-ac54fb743e43","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:49:28.259Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"fc1395d3-54d9-40e7-a5db-ac54fb743e43"}
{"level":"info","ts":"2026-06-05T09:49:28.261Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"fc1395d3-54d9-40e7-a5db-ac54fb743e43","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:48:58Z","endTime":"2026-06-05T09:49:21Z"},"jobId":"stop-rnf5t","rayClusterName":"stop-z5q6j","dashboardURL":"stop-z5q6j-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-06-05T09:48:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:48:50Z","stateTransitionTimes":{"ready":"2026-06-05T09:48:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.25","serviceIP":"10.132.0.25","podName":"stop-z5q6j-head-kpbk9","serviceName":"stop-z5q6j-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:48:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:48:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:48:58Z","endTime":"2026-06-05T09:49:21Z"},"jobId":"stop-rnf5t","rayClusterName":"stop-z5q6j","dashboardURL":"stop-z5q6j-head-svc.test-ns-thrrw.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Complete","message":"Job was intentionally stopped.","startTime":"2026-06-05T09:48:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:48:50Z","stateTransitionTimes":{"ready":"2026-06-05T09:48:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.25","serviceIP":"10.132.0.25","podName":"stop-z5q6j-head-kpbk9","serviceName":"stop-z5q6j-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:48:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:48:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:48:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:48:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:49:28.262Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"fc1395d3-54d9-40e7-a5db-ac54fb743e43","old JobStatus":"STOPPED","new JobStatus":"STOPPED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Complete"}
{"level":"info","ts":"2026-06-05T09:49:28.282Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"85482486-d2b4-4ec5-a705-c4a36c304dea","JobStatus":"STOPPED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:28.282Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"85482486-d2b4-4ec5-a705-c4a36c304dea","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-05 09:49:28 +0000 UTC","Now":"2026-06-05T09:49:28.282Z","ShutdownTime":"2026-06-05T09:49:28.000Z"}
{"level":"info","ts":"2026-06-05T09:49:28.711Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"d1ab1af6-ace7-4ac8-993c-f50118868948","DeletionTimestamp":"2026-06-05 09:49:28 +0000 UTC"}
{"level":"info","ts":"2026-06-05T09:49:28.711Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"d1ab1af6-ace7-4ac8-993c-f50118868948","finalizer":"ray.io/rayjob-finalizer"}
{"level":"error","ts":"2026-06-05T09:49:28.728Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-thrrw"},"reconcileID":"d4f13036-4276-4a96-bb1a-650d1055a06f","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:28.728Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-thrrw"},"reconcileID":"d4f13036-4276-4a96-bb1a-650d1055a06f"}
{"level":"error","ts":"2026-06-05T09:49:28.728Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-thrrw"},"reconcileID":"d4f13036-4276-4a96-bb1a-650d1055a06f","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:28.729Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"6602cfbd-72ed-40d2-b69d-5e3745426ba1"}
{"level":"error","ts":"2026-06-05T09:49:28.733Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-thrrw"},"reconcileID":"0ce7b09e-5cbc-4c86-b859-5a8a4e2a67d2","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:28.733Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-thrrw"},"reconcileID":"0ce7b09e-5cbc-4c86-b859-5a8a4e2a67d2"}
{"level":"error","ts":"2026-06-05T09:49:28.733Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-thrrw"},"reconcileID":"0ce7b09e-5cbc-4c86-b859-5a8a4e2a67d2","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-05T09:49:28.744Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-thrrw"},"reconcileID":"bc72bb47-2315-4e77-b96a-073f095e1387","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:28.744Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-thrrw"},"reconcileID":"bc72bb47-2315-4e77-b96a-073f095e1387"}
{"level":"error","ts":"2026-06-05T09:49:28.744Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-thrrw"},"reconcileID":"bc72bb47-2315-4e77-b96a-073f095e1387","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:28.744Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"e7fda414-83a1-43ad-9150-e9521159c4f3"}
{"level":"info","ts":"2026-06-05T09:49:28.751Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"7f0f7403-a3e4-439b-9767-6b24e2c809f2","rayCluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:49:28.751Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"8cfa4731-12c5-4cd5-8420-808d6512c83b","namespacedName":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:49:28.751Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"8cfa4731-12c5-4cd5-8420-808d6512c83b","cluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:49:28.751Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"8cfa4731-12c5-4cd5-8420-808d6512c83b","cluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:49:28.751Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"reconcileID":"e370c3c6-3e8b-46fd-a12e-070d7f85a617"}
{"level":"info","ts":"2026-06-05T09:49:28.751Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"7f0f7403-a3e4-439b-9767-6b24e2c809f2","pod":"stop-z5q6j-head-kpbk9","ip":"10.132.0.25"}
{"level":"info","ts":"2026-06-05T09:49:28.751Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"7f0f7403-a3e4-439b-9767-6b24e2c809f2","pod":"stop-z5q6j-small-group-worker-kjlsx","ip":"10.133.0.51"}
{"level":"info","ts":"2026-06-05T09:49:28.751Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"5e61d31e-91b1-4599-9656-25a0fb100486"}
{"level":"info","ts":"2026-06-05T09:49:28.751Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"8cfa4731-12c5-4cd5-8420-808d6512c83b","namespace":"test-ns-thrrw","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-05T09:49:28.751Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"7f0f7403-a3e4-439b-9767-6b24e2c809f2","rayCluster":"stop-z5q6j","podIPs":["10.132.0.25","10.133.0.51"]}
{"level":"info","ts":"2026-06-05T09:49:28.751Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"cc3d4080-1194-4dbd-822d-2756bbc07f73"}
{"level":"info","ts":"2026-06-05T09:49:28.759Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"8cfa4731-12c5-4cd5-8420-808d6512c83b","name":"kuberay-gateway-access","namespace":"test-ns-thrrw"}
{"level":"error","ts":"2026-06-05T09:49:28.765Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-thrrw"},"reconcileID":"bfce7603-0cd4-49fc-9e01-cff8ab49c1f0","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:28.765Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-thrrw"},"reconcileID":"bfce7603-0cd4-49fc-9e01-cff8ab49c1f0"}
{"level":"error","ts":"2026-06-05T09:49:28.765Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-thrrw"},"reconcileID":"bfce7603-0cd4-49fc-9e01-cff8ab49c1f0","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:28.772Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"8cfa4731-12c5-4cd5-8420-808d6512c83b","name":"test-ns-thrrw-stop-z5q6j","namespace":"opendatahub"}
{"level":"info","ts":"2026-06-05T09:49:28.779Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"8cfa4731-12c5-4cd5-8420-808d6512c83b","configMap":"kube-rbac-proxy-config-stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:49:28.787Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"8cfa4731-12c5-4cd5-8420-808d6512c83b","serviceAccount":"stop-z5q6j-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:49:28.795Z","logger":"raycluster-resource","msg":"default","name":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:49:28.795Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"stop-z5q6j","namespace":"test-ns-thrrw"}
{"level":"error","ts":"2026-06-05T09:49:28.805Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-thrrw"},"reconcileID":"89fc415e-cf52-4f1b-b0ee-be525798009f","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:28.806Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-thrrw"},"reconcileID":"89fc415e-cf52-4f1b-b0ee-be525798009f"}
{"level":"error","ts":"2026-06-05T09:49:28.806Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-thrrw"},"reconcileID":"89fc415e-cf52-4f1b-b0ee-be525798009f","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:28.818Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"8cfa4731-12c5-4cd5-8420-808d6512c83b","cluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:49:28.818Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"e35a46eb-c337-4e7f-9bc4-718e0f66b04b","namespacedName":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:49:28.818Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"e35a46eb-c337-4e7f-9bc4-718e0f66b04b","cluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:49:28.818Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"e35a46eb-c337-4e7f-9bc4-718e0f66b04b","cluster":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:49:28.818Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"e35a46eb-c337-4e7f-9bc4-718e0f66b04b","namespace":"test-ns-thrrw","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-05T09:49:28.818Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"e35a46eb-c337-4e7f-9bc4-718e0f66b04b","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-06-05T09:49:28.819Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"9674f340-9d2b-4c76-8f76-a5f884c2e60e"}
{"level":"info","ts":"2026-06-05T09:49:28.819Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"2eabe7ff-8b52-4a66-b55e-4daa20b4967d"}
{"level":"info","ts":"2026-06-05T09:49:28.819Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"b78a06e2-0dd5-43f5-905d-0cf560f60606","namespace":"test-ns-thrrw","clusterName":"stop-z5q6j"}
{"level":"error","ts":"2026-06-05T09:49:28.823Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"e35a46eb-c337-4e7f-9bc4-718e0f66b04b","error":"rayclusters.ray.io \"stop-z5q6j\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:214\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:155\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:28.823Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"e35a46eb-c337-4e7f-9bc4-718e0f66b04b"}
{"level":"error","ts":"2026-06-05T09:49:28.823Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"e35a46eb-c337-4e7f-9bc4-718e0f66b04b","error":"rayclusters.ray.io \"stop-z5q6j\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:28.823Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"52d4b10e-9ab0-4b68-8bcd-a6c26d49e973","namespacedName":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:49:28.823Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"52d4b10e-9ab0-4b68-8bcd-a6c26d49e973"}
{"level":"info","ts":"2026-06-05T09:49:28.823Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"52d4b10e-9ab0-4b68-8bcd-a6c26d49e973","namespace":"test-ns-thrrw","authClusters":0}
{"level":"info","ts":"2026-06-05T09:49:28.823Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"52d4b10e-9ab0-4b68-8bcd-a6c26d49e973","namespace":"test-ns-thrrw"}
{"level":"info","ts":"2026-06-05T09:49:28.828Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"9e34d90c-ea84-4276-8147-3f4bcf762453"}
{"level":"info","ts":"2026-06-05T09:49:28.829Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"1d0e9e1b-eb59-4540-a566-0f2ae98f6368","namespacedName":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:49:28.829Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"1d0e9e1b-eb59-4540-a566-0f2ae98f6368"}
{"level":"info","ts":"2026-06-05T09:49:28.829Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"1d0e9e1b-eb59-4540-a566-0f2ae98f6368","namespace":"test-ns-thrrw","authClusters":0}
{"level":"info","ts":"2026-06-05T09:49:28.829Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"1d0e9e1b-eb59-4540-a566-0f2ae98f6368","namespace":"test-ns-thrrw"}
{"level":"info","ts":"2026-06-05T09:49:28.833Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"4c61f20e-80da-4158-9230-19974e35af2f"}
{"level":"info","ts":"2026-06-05T09:49:28.833Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"5ef13fb9-6af2-46da-bfb5-764935cb550d","namespacedName":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:49:28.834Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"5ef13fb9-6af2-46da-bfb5-764935cb550d"}
{"level":"info","ts":"2026-06-05T09:49:28.834Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"5ef13fb9-6af2-46da-bfb5-764935cb550d","namespace":"test-ns-thrrw","authClusters":0}
{"level":"info","ts":"2026-06-05T09:49:28.834Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"5ef13fb9-6af2-46da-bfb5-764935cb550d","namespace":"test-ns-thrrw"}
{"level":"error","ts":"2026-06-05T09:49:28.846Z","msg":"Failed to delete head certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"b78a06e2-0dd5-43f5-905d-0cf560f60606","certificate":"ray-head-cert-stop-z5q6j","error":"certificates.cert-manager.io \"ray-head-cert-stop-z5q6j\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:261\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-05T09:49:28.859Z","msg":"Failed to delete worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"b78a06e2-0dd5-43f5-905d-0cf560f60606","certificate":"ray-worker-cert-stop-z5q6j","error":"certificates.cert-manager.io \"ray-worker-cert-stop-z5q6j\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:268\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-05T09:49:28.906Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-thrrw"},"reconcileID":"5109c2aa-a35c-437a-9205-531d25ef76d9","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:28.906Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-thrrw"},"reconcileID":"5109c2aa-a35c-437a-9205-531d25ef76d9"}
{"level":"error","ts":"2026-06-05T09:49:28.906Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-thrrw"},"reconcileID":"5109c2aa-a35c-437a-9205-531d25ef76d9","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:28.919Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"b78a06e2-0dd5-43f5-905d-0cf560f60606","errorCount":2}
{"level":"info","ts":"2026-06-05T09:49:28.919Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"b78a06e2-0dd5-43f5-905d-0cf560f60606"}
{"level":"error","ts":"2026-06-05T09:49:28.919Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"b78a06e2-0dd5-43f5-905d-0cf560f60606","error":"certificates.cert-manager.io \"ray-head-cert-stop-z5q6j\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:28.924Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"992bb15a-d6d5-4741-b7df-f896b1831c48","namespace":"test-ns-thrrw","clusterName":"stop-z5q6j"}
{"level":"info","ts":"2026-06-05T09:49:28.924Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-z5q6j","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"stop-z5q6j","reconcileID":"992bb15a-d6d5-4741-b7df-f896b1831c48","namespace":"test-ns-thrrw","clusterName":"stop-z5q6j"}
{"level":"error","ts":"2026-06-05T09:49:29.067Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-thrrw"},"reconcileID":"c5ac5572-ca35-4349-9c05-5ee583f963e3","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:29.067Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-thrrw"},"reconcileID":"c5ac5572-ca35-4349-9c05-5ee583f963e3"}
{"level":"error","ts":"2026-06-05T09:49:29.067Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-thrrw"},"reconcileID":"c5ac5572-ca35-4349-9c05-5ee583f963e3","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-05T09:49:29.388Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-thrrw"},"reconcileID":"7b174981-254e-452d-8483-cc8301a47cd0","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:29.388Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-thrrw"},"reconcileID":"7b174981-254e-452d-8483-cc8301a47cd0"}
{"level":"error","ts":"2026-06-05T09:49:29.388Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-thrrw"},"reconcileID":"7b174981-254e-452d-8483-cc8301a47cd0","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-05T09:49:30.028Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-thrrw"},"reconcileID":"148be2fa-5141-4dc5-875b-56e74e2481bd","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:30.028Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-thrrw"},"reconcileID":"148be2fa-5141-4dc5-875b-56e74e2481bd"}
{"level":"error","ts":"2026-06-05T09:49:30.028Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-thrrw"},"reconcileID":"148be2fa-5141-4dc5-875b-56e74e2481bd","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:30.158Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"stop","namespace":"test-ns-thrrw"},"reconcileID":"6b562aef-8c9c-43c4-a504-7696f3f636ba"}
{"level":"error","ts":"2026-06-05T09:49:31.309Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-thrrw"},"reconcileID":"93238c8d-2d8a-48d3-9a55-e86a1fb9bb22","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:31.309Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-thrrw"},"reconcileID":"93238c8d-2d8a-48d3-9a55-e86a1fb9bb22"}
{"level":"error","ts":"2026-06-05T09:49:31.309Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-thrrw"},"reconcileID":"93238c8d-2d8a-48d3-9a55-e86a1fb9bb22","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:33.739Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"0ee0382d-1e12-46b7-9c23-edeb63d0d162","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:33.739Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"0ee0382d-1e12-46b7-9c23-edeb63d0d162","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-05T09:49:33.753Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"0ee0382d-1e12-46b7-9c23-edeb63d0d162"}
{"level":"info","ts":"2026-06-05T09:49:33.753Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"0ee0382d-1e12-46b7-9c23-edeb63d0d162","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-05T09:49:33.753Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"0ee0382d-1e12-46b7-9c23-edeb63d0d162","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-77jkz","rayClusterName":"long-running-wg4gd","jobDeploymentStatus":"Initializing","startTime":"2026-06-05T09:49:33Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:49:33.753Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"0ee0382d-1e12-46b7-9c23-edeb63d0d162","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-05T09:49:33.767Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"36d9e559-354f-457d-83ea-e17bf6d29245","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:33.767Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"36d9e559-354f-457d-83ea-e17bf6d29245"}
{"level":"info","ts":"2026-06-05T09:49:33.767Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"36d9e559-354f-457d-83ea-e17bf6d29245","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-05T09:49:33.767Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"36d9e559-354f-457d-83ea-e17bf6d29245","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-k99dx","rayClusterName":"long-running-v7npw","jobDeploymentStatus":"Initializing","startTime":"2026-06-05T09:49:33Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:49:33.767Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"36d9e559-354f-457d-83ea-e17bf6d29245","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-05T09:49:33.774Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"36d9e559-354f-457d-83ea-e17bf6d29245","error":"Operation cannot be fulfilled on rayjobs.ray.io \"long-running\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-06-05T09:49:33.774Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"36d9e559-354f-457d-83ea-e17bf6d29245"}
{"level":"error","ts":"2026-06-05T09:49:33.774Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"36d9e559-354f-457d-83ea-e17bf6d29245","error":"Operation cannot be fulfilled on rayjobs.ray.io \"long-running\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:33.774Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"fe1842e3-f7ea-464b-ad43-fd1686b61d48","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:33.774Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"fe1842e3-f7ea-464b-ad43-fd1686b61d48","name":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:33.774Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"fe1842e3-f7ea-464b-ad43-fd1686b61d48","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:49:33.774Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"fe1842e3-f7ea-464b-ad43-fd1686b61d48","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:49:33.779Z","logger":"raycluster-resource","msg":"default","name":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:33.779Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-wg4gd","namespace":"test-ns-thrrw"}
{"level":"info","ts":"2026-06-05T09:49:33.796Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"fe1842e3-f7ea-464b-ad43-fd1686b61d48","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:49:33.796Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"26260f42-8334-464d-a4d0-7b2ec5e4eced"}
{"level":"info","ts":"2026-06-05T09:49:33.796Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"26260f42-8334-464d-a4d0-7b2ec5e4eced"}
{"level":"info","ts":"2026-06-05T09:49:33.796Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"3dee508a-04ce-4797-81f9-072192ab3122","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:33.796Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"fe1842e3-f7ea-464b-ad43-fd1686b61d48"}
{"level":"info","ts":"2026-06-05T09:49:33.796Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"fe1842e3-f7ea-464b-ad43-fd1686b61d48","RayCluster":"long-running-wg4gd","State":""}
{"level":"info","ts":"2026-06-05T09:49:33.796Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"fac5e3c7-bc55-4a48-b118-69d32a053033","namespacedName":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:49:33.797Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"fac5e3c7-bc55-4a48-b118-69d32a053033","mode":"IntegratedOAuth","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:33.797Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"fac5e3c7-bc55-4a48-b118-69d32a053033","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:33.797Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"bac908f9-07a3-4c42-87a9-c8d1760edce9","rayCluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:33.797Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"fac5e3c7-bc55-4a48-b118-69d32a053033","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:33.797Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"bac908f9-07a3-4c42-87a9-c8d1760edce9","rayCluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:33.797Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"0284a209-25b1-485b-87be-dc7325cc4eac","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:33.797Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"0284a209-25b1-485b-87be-dc7325cc4eac","name":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:33.797Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"0284a209-25b1-485b-87be-dc7325cc4eac","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:49:33.797Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"0284a209-25b1-485b-87be-dc7325cc4eac"}
{"level":"info","ts":"2026-06-05T09:49:33.797Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"0284a209-25b1-485b-87be-dc7325cc4eac","RayCluster":"long-running-wg4gd","State":""}
{"level":"info","ts":"2026-06-05T09:49:33.797Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"da9f8786-f2a8-4a18-8117-7ba7c1f7ee8e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:33.797Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"da9f8786-f2a8-4a18-8117-7ba7c1f7ee8e","name":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:33.797Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"da9f8786-f2a8-4a18-8117-7ba7c1f7ee8e","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:49:33.797Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"da9f8786-f2a8-4a18-8117-7ba7c1f7ee8e"}
{"level":"info","ts":"2026-06-05T09:49:33.797Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"da9f8786-f2a8-4a18-8117-7ba7c1f7ee8e","RayCluster":"long-running-wg4gd","State":""}
{"level":"info","ts":"2026-06-05T09:49:33.803Z","logger":"raycluster-resource","msg":"default","name":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:33.803Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-wg4gd","namespace":"test-ns-thrrw"}
{"level":"info","ts":"2026-06-05T09:49:33.805Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"26260f42-8334-464d-a4d0-7b2ec5e4eced","name":"long-running-wg4gd-head-svc"}
{"level":"info","ts":"2026-06-05T09:49:33.805Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"26260f42-8334-464d-a4d0-7b2ec5e4eced","error":"head certificate secret ray-head-secret-long-running-wg4gd not found"}
{"level":"error","ts":"2026-06-05T09:49:33.805Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"26260f42-8334-464d-a4d0-7b2ec5e4eced","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-wg4gd not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:33.805Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"26260f42-8334-464d-a4d0-7b2ec5e4eced","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-wg4gd not found"}
{"level":"info","ts":"2026-06-05T09:49:33.805Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"26260f42-8334-464d-a4d0-7b2ec5e4eced","filter labels":["test-ns-thrrw",{"ray.io/cluster":"long-running-wg4gd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:49:33.805Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"26260f42-8334-464d-a4d0-7b2ec5e4eced","filter labels":["test-ns-thrrw",{"ray.io/cluster":"long-running-wg4gd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:49:33.805Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"26260f42-8334-464d-a4d0-7b2ec5e4eced","filter labels":["test-ns-thrrw",{"ray.io/cluster":"long-running-wg4gd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:49:33.805Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"26260f42-8334-464d-a4d0-7b2ec5e4eced","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-05T09:49:33.805Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"26260f42-8334-464d-a4d0-7b2ec5e4eced","name":"long-running-wg4gd","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:49:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-wg4gd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:49:33.808Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"bac908f9-07a3-4c42-87a9-c8d1760edce9","rayCluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:33.808Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"bac908f9-07a3-4c42-87a9-c8d1760edce9","rayCluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:33.810Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"ba77cb26-0b58-4ac4-97a6-a8ec644a427b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:33.810Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"ba77cb26-0b58-4ac4-97a6-a8ec644a427b","name":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:33.810Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"ba77cb26-0b58-4ac4-97a6-a8ec644a427b","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:49:33.810Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"ba77cb26-0b58-4ac4-97a6-a8ec644a427b"}
{"level":"info","ts":"2026-06-05T09:49:33.810Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"ba77cb26-0b58-4ac4-97a6-a8ec644a427b","RayCluster":"long-running-wg4gd","State":""}
{"level":"info","ts":"2026-06-05T09:49:33.813Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"3dee508a-04ce-4797-81f9-072192ab3122","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:33.813Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"f78d1f42-0b1a-4d14-8318-804362aab988","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:33.819Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"fac5e3c7-bc55-4a48-b118-69d32a053033","name":"long-running-wg4gd-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:49:33.822Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"26260f42-8334-464d-a4d0-7b2ec5e4eced","name":"long-running-wg4gd","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-wg4gd\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-thrrw","name":"long-running-wg4gd"}}
{"level":"info","ts":"2026-06-05T09:49:33.822Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"26260f42-8334-464d-a4d0-7b2ec5e4eced"}
{"level":"error","ts":"2026-06-05T09:49:33.822Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"26260f42-8334-464d-a4d0-7b2ec5e4eced","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-wg4gd not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:33.822Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7b8000b4-e9a5-409c-be1e-b64cb4431ff8"}
{"level":"info","ts":"2026-06-05T09:49:33.822Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7b8000b4-e9a5-409c-be1e-b64cb4431ff8"}
{"level":"info","ts":"2026-06-05T09:49:33.822Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7b8000b4-e9a5-409c-be1e-b64cb4431ff8","1 head service found":"long-running-wg4gd-head-svc"}
{"level":"info","ts":"2026-06-05T09:49:33.822Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7b8000b4-e9a5-409c-be1e-b64cb4431ff8","error":"head certificate secret ray-head-secret-long-running-wg4gd not found"}
{"level":"error","ts":"2026-06-05T09:49:33.822Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7b8000b4-e9a5-409c-be1e-b64cb4431ff8","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-wg4gd not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:33.822Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7b8000b4-e9a5-409c-be1e-b64cb4431ff8","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-wg4gd not found"}
{"level":"info","ts":"2026-06-05T09:49:33.822Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7b8000b4-e9a5-409c-be1e-b64cb4431ff8","filter labels":["test-ns-thrrw",{"ray.io/cluster":"long-running-wg4gd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:49:33.822Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7b8000b4-e9a5-409c-be1e-b64cb4431ff8","filter labels":["test-ns-thrrw",{"ray.io/cluster":"long-running-wg4gd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:49:33.822Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7b8000b4-e9a5-409c-be1e-b64cb4431ff8","filter labels":["test-ns-thrrw",{"ray.io/cluster":"long-running-wg4gd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:49:33.822Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7b8000b4-e9a5-409c-be1e-b64cb4431ff8","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-05T09:49:33.822Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7b8000b4-e9a5-409c-be1e-b64cb4431ff8","name":"long-running-wg4gd","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:49:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-wg4gd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:49:33.823Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"bac908f9-07a3-4c42-87a9-c8d1760edce9"}
{"level":"info","ts":"2026-06-05T09:49:33.823Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"bac908f9-07a3-4c42-87a9-c8d1760edce9","rayCluster":"long-running-wg4gd","secret":"ca-secret"}
{"level":"info","ts":"2026-06-05T09:49:33.823Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"bac908f9-07a3-4c42-87a9-c8d1760edce9","rayCluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:33.832Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"f78d1f42-0b1a-4d14-8318-804362aab988","name":"long-running-wg4gd-head"}
{"level":"info","ts":"2026-06-05T09:49:33.832Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"fac5e3c7-bc55-4a48-b118-69d32a053033","name":"kuberay-gateway-access","namespace":"test-ns-thrrw","operation":"created"}
{"level":"info","ts":"2026-06-05T09:49:33.834Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7b8000b4-e9a5-409c-be1e-b64cb4431ff8"}
{"level":"error","ts":"2026-06-05T09:49:33.834Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7b8000b4-e9a5-409c-be1e-b64cb4431ff8","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-wg4gd not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:33.834Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7e134413-2d44-46e7-a22e-42987ebef3c6"}
{"level":"info","ts":"2026-06-05T09:49:33.834Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7e134413-2d44-46e7-a22e-42987ebef3c6"}
{"level":"info","ts":"2026-06-05T09:49:33.834Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7e134413-2d44-46e7-a22e-42987ebef3c6","1 head service found":"long-running-wg4gd-head-svc"}
{"level":"info","ts":"2026-06-05T09:49:33.834Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7e134413-2d44-46e7-a22e-42987ebef3c6","error":"head certificate secret ray-head-secret-long-running-wg4gd not found"}
{"level":"error","ts":"2026-06-05T09:49:33.834Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7e134413-2d44-46e7-a22e-42987ebef3c6","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-wg4gd not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:33.834Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7e134413-2d44-46e7-a22e-42987ebef3c6","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-wg4gd not found"}
{"level":"info","ts":"2026-06-05T09:49:33.834Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7e134413-2d44-46e7-a22e-42987ebef3c6","filter labels":["test-ns-thrrw",{"ray.io/cluster":"long-running-wg4gd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:49:33.834Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7e134413-2d44-46e7-a22e-42987ebef3c6","filter labels":["test-ns-thrrw",{"ray.io/cluster":"long-running-wg4gd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:49:33.834Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7e134413-2d44-46e7-a22e-42987ebef3c6","filter labels":["test-ns-thrrw",{"ray.io/cluster":"long-running-wg4gd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:49:33.834Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7e134413-2d44-46e7-a22e-42987ebef3c6","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-05T09:49:33.834Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7e134413-2d44-46e7-a22e-42987ebef3c6","name":"long-running-wg4gd","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:49:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-wg4gd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"error","ts":"2026-06-05T09:49:33.906Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-thrrw"},"reconcileID":"12630f83-4a26-425e-b015-0c4529082162","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:33.906Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-thrrw"},"reconcileID":"12630f83-4a26-425e-b015-0c4529082162"}
{"level":"error","ts":"2026-06-05T09:49:33.906Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-thrrw"},"reconcileID":"12630f83-4a26-425e-b015-0c4529082162","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:33.906Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"bac908f9-07a3-4c42-87a9-c8d1760edce9","rayCluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:33.906Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"fac5e3c7-bc55-4a48-b118-69d32a053033","name":"test-ns-thrrw-long-running-wg4gd","operation":"created"}
{"level":"info","ts":"2026-06-05T09:49:33.912Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"f78d1f42-0b1a-4d14-8318-804362aab988","name":"long-running-wg4gd-workers"}
{"level":"info","ts":"2026-06-05T09:49:33.912Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"f78d1f42-0b1a-4d14-8318-804362aab988","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:33.912Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"2a969e25-74f8-47d3-b586-3ee25f209870","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:33.912Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"fac5e3c7-bc55-4a48-b118-69d32a053033","name":"kube-rbac-proxy-config-long-running-wg4gd","operation":"created"}
{"level":"info","ts":"2026-06-05T09:49:33.914Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7e134413-2d44-46e7-a22e-42987ebef3c6","name":"long-running-wg4gd","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-wg4gd\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-thrrw","name":"long-running-wg4gd"}}
{"level":"info","ts":"2026-06-05T09:49:33.914Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7e134413-2d44-46e7-a22e-42987ebef3c6"}
{"level":"error","ts":"2026-06-05T09:49:33.914Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7e134413-2d44-46e7-a22e-42987ebef3c6","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-wg4gd not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:33.914Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"fbecd5cd-06f4-46c8-a82e-8eab0cf042ab"}
{"level":"info","ts":"2026-06-05T09:49:33.914Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"fbecd5cd-06f4-46c8-a82e-8eab0cf042ab"}
{"level":"info","ts":"2026-06-05T09:49:33.914Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"fbecd5cd-06f4-46c8-a82e-8eab0cf042ab","1 head service found":"long-running-wg4gd-head-svc"}
{"level":"info","ts":"2026-06-05T09:49:33.914Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"fbecd5cd-06f4-46c8-a82e-8eab0cf042ab","error":"head certificate secret ray-head-secret-long-running-wg4gd not found"}
{"level":"error","ts":"2026-06-05T09:49:33.914Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"fbecd5cd-06f4-46c8-a82e-8eab0cf042ab","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-wg4gd not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:33.914Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"fbecd5cd-06f4-46c8-a82e-8eab0cf042ab","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-wg4gd not found"}
{"level":"info","ts":"2026-06-05T09:49:33.914Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"fbecd5cd-06f4-46c8-a82e-8eab0cf042ab","filter labels":["test-ns-thrrw",{"ray.io/cluster":"long-running-wg4gd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:49:33.914Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"fbecd5cd-06f4-46c8-a82e-8eab0cf042ab","filter labels":["test-ns-thrrw",{"ray.io/cluster":"long-running-wg4gd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:49:33.914Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"fbecd5cd-06f4-46c8-a82e-8eab0cf042ab","filter labels":["test-ns-thrrw",{"ray.io/cluster":"long-running-wg4gd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:49:33.914Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"fbecd5cd-06f4-46c8-a82e-8eab0cf042ab"}
{"level":"error","ts":"2026-06-05T09:49:33.914Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"fbecd5cd-06f4-46c8-a82e-8eab0cf042ab","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-wg4gd not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:33.918Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"bac908f9-07a3-4c42-87a9-c8d1760edce9"}
{"level":"info","ts":"2026-06-05T09:49:33.918Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"bac908f9-07a3-4c42-87a9-c8d1760edce9","rayCluster":"long-running-wg4gd","podIPs":[]}
{"level":"error","ts":"2026-06-05T09:49:33.919Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"fac5e3c7-bc55-4a48-b118-69d32a053033","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-wg4gd\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:33.919Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"fac5e3c7-bc55-4a48-b118-69d32a053033"}
{"level":"error","ts":"2026-06-05T09:49:33.919Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"fac5e3c7-bc55-4a48-b118-69d32a053033","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-wg4gd\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:33.919Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"ecc4a0fa-7049-4506-a734-ed82b665b376","namespacedName":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:49:33.919Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"ecc4a0fa-7049-4506-a734-ed82b665b376","mode":"IntegratedOAuth","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:33.919Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"ecc4a0fa-7049-4506-a734-ed82b665b376","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:33.925Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"2a969e25-74f8-47d3-b586-3ee25f209870","name":"long-running-wg4gd-head"}
{"level":"info","ts":"2026-06-05T09:49:34.005Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"40c4154a-2b59-4111-b071-b6263986e1bd"}
{"level":"info","ts":"2026-06-05T09:49:34.005Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"40c4154a-2b59-4111-b071-b6263986e1bd"}
{"level":"info","ts":"2026-06-05T09:49:34.006Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"bac908f9-07a3-4c42-87a9-c8d1760edce9"}
{"level":"info","ts":"2026-06-05T09:49:34.006Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"40c4154a-2b59-4111-b071-b6263986e1bd","1 head service found":"long-running-wg4gd-head-svc"}
{"level":"info","ts":"2026-06-05T09:49:34.006Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"40c4154a-2b59-4111-b071-b6263986e1bd","error":"head certificate secret ray-head-secret-long-running-wg4gd not found"}
{"level":"info","ts":"2026-06-05T09:49:34.006Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"41ad6b28-4420-4197-9fe7-0a4af70d1de5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:34.006Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"41ad6b28-4420-4197-9fe7-0a4af70d1de5","name":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:34.006Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"41ad6b28-4420-4197-9fe7-0a4af70d1de5","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"}}
{"level":"error","ts":"2026-06-05T09:49:34.006Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"40c4154a-2b59-4111-b071-b6263986e1bd","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-wg4gd not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:34.006Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"40c4154a-2b59-4111-b071-b6263986e1bd","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-wg4gd not found"}
{"level":"info","ts":"2026-06-05T09:49:34.006Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"40c4154a-2b59-4111-b071-b6263986e1bd","filter labels":["test-ns-thrrw",{"ray.io/cluster":"long-running-wg4gd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:49:34.006Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"bac908f9-07a3-4c42-87a9-c8d1760edce9","rayCluster":"long-running-wg4gd","podIPs":[]}
{"level":"info","ts":"2026-06-05T09:49:34.006Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"40c4154a-2b59-4111-b071-b6263986e1bd","filter labels":["test-ns-thrrw",{"ray.io/cluster":"long-running-wg4gd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:49:34.006Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"40c4154a-2b59-4111-b071-b6263986e1bd","filter labels":["test-ns-thrrw",{"ray.io/cluster":"long-running-wg4gd","ray.io/node-type":"head"}]}
{"level":"error","ts":"2026-06-05T09:49:34.006Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"bac908f9-07a3-4c42-87a9-c8d1760edce9","error":"Certificate.cert-manager.io \"ray-worker-cert-long-running-wg4gd\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:34.006Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"40c4154a-2b59-4111-b071-b6263986e1bd"}
{"level":"info","ts":"2026-06-05T09:49:34.006Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"bac908f9-07a3-4c42-87a9-c8d1760edce9"}
{"level":"error","ts":"2026-06-05T09:49:34.006Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"40c4154a-2b59-4111-b071-b6263986e1bd","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-wg4gd not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-05T09:49:34.006Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"bac908f9-07a3-4c42-87a9-c8d1760edce9","error":"Certificate.cert-manager.io \"ray-worker-cert-long-running-wg4gd\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:34.006Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"41ad6b28-4420-4197-9fe7-0a4af70d1de5"}
{"level":"info","ts":"2026-06-05T09:49:34.006Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"41ad6b28-4420-4197-9fe7-0a4af70d1de5","RayCluster":"long-running-wg4gd","State":""}
{"level":"info","ts":"2026-06-05T09:49:34.006Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"f29a3eda-7028-40d0-b407-1bb081e56852","rayCluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:34.006Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"f29a3eda-7028-40d0-b407-1bb081e56852"}
{"level":"info","ts":"2026-06-05T09:49:34.006Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"ecc4a0fa-7049-4506-a734-ed82b665b376","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:34.006Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"328f65ca-d05f-4441-b24d-9b5e9b54cd24","namespacedName":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:49:34.006Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"328f65ca-d05f-4441-b24d-9b5e9b54cd24","mode":"IntegratedOAuth","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:34.006Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"328f65ca-d05f-4441-b24d-9b5e9b54cd24","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:34.006Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"b2e39d50-b4ab-46f8-b4f1-48cb31701773","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:34.006Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"b2e39d50-b4ab-46f8-b4f1-48cb31701773","name":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:34.007Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"b2e39d50-b4ab-46f8-b4f1-48cb31701773","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:49:34.007Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"b2e39d50-b4ab-46f8-b4f1-48cb31701773"}
{"level":"info","ts":"2026-06-05T09:49:34.007Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"b2e39d50-b4ab-46f8-b4f1-48cb31701773","RayCluster":"long-running-wg4gd","State":""}
{"level":"info","ts":"2026-06-05T09:49:34.011Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"2a969e25-74f8-47d3-b586-3ee25f209870","name":"long-running-wg4gd-workers"}
{"level":"info","ts":"2026-06-05T09:49:34.011Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"2a969e25-74f8-47d3-b586-3ee25f209870","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:34.011Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"494e09f5-1dd2-4fff-8eb1-b3987e767e89","cluster":"long-running-wg4gd"}
{"level":"error","ts":"2026-06-05T09:49:34.014Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"328f65ca-d05f-4441-b24d-9b5e9b54cd24","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-wg4gd\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:34.014Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"328f65ca-d05f-4441-b24d-9b5e9b54cd24"}
{"level":"error","ts":"2026-06-05T09:49:34.014Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"328f65ca-d05f-4441-b24d-9b5e9b54cd24","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-wg4gd\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:34.020Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"833edbe0-8ec0-4cf2-91e8-d2e82db91582","namespacedName":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:49:34.020Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"833edbe0-8ec0-4cf2-91e8-d2e82db91582","mode":"IntegratedOAuth","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:34.020Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"833edbe0-8ec0-4cf2-91e8-d2e82db91582","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:34.022Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"494e09f5-1dd2-4fff-8eb1-b3987e767e89","name":"long-running-wg4gd-head"}
{"level":"info","ts":"2026-06-05T09:49:34.030Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"833edbe0-8ec0-4cf2-91e8-d2e82db91582","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:34.105Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"e2d251ea-c329-422e-8e18-d377dc9fd3a8","rayCluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:34.106Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"e2d251ea-c329-422e-8e18-d377dc9fd3a8"}
{"level":"info","ts":"2026-06-05T09:49:34.106Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"494e09f5-1dd2-4fff-8eb1-b3987e767e89","name":"long-running-wg4gd-workers"}
{"level":"info","ts":"2026-06-05T09:49:34.106Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"494e09f5-1dd2-4fff-8eb1-b3987e767e89","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:34.106Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"e4dea4e1-d9df-4928-bec7-1f95de04f8c5","rayCluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:34.106Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"e4dea4e1-d9df-4928-bec7-1f95de04f8c5"}
{"level":"info","ts":"2026-06-05T09:49:34.106Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"4482ae2a-04bf-476d-bfeb-214fcb72d4db"}
{"level":"info","ts":"2026-06-05T09:49:34.106Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"4482ae2a-04bf-476d-bfeb-214fcb72d4db"}
{"level":"info","ts":"2026-06-05T09:49:34.106Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"4482ae2a-04bf-476d-bfeb-214fcb72d4db","1 head service found":"long-running-wg4gd-head-svc"}
{"level":"info","ts":"2026-06-05T09:49:34.106Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"4482ae2a-04bf-476d-bfeb-214fcb72d4db","error":"head certificate secret ray-head-secret-long-running-wg4gd not found"}
{"level":"error","ts":"2026-06-05T09:49:34.106Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"4482ae2a-04bf-476d-bfeb-214fcb72d4db","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-wg4gd not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:34.106Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"4482ae2a-04bf-476d-bfeb-214fcb72d4db","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-wg4gd not found"}
{"level":"info","ts":"2026-06-05T09:49:34.106Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"4482ae2a-04bf-476d-bfeb-214fcb72d4db","filter labels":["test-ns-thrrw",{"ray.io/cluster":"long-running-wg4gd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:49:34.106Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"4482ae2a-04bf-476d-bfeb-214fcb72d4db","filter labels":["test-ns-thrrw",{"ray.io/cluster":"long-running-wg4gd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:49:34.106Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"4482ae2a-04bf-476d-bfeb-214fcb72d4db","filter labels":["test-ns-thrrw",{"ray.io/cluster":"long-running-wg4gd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:49:34.106Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"4482ae2a-04bf-476d-bfeb-214fcb72d4db"}
{"level":"error","ts":"2026-06-05T09:49:34.106Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"4482ae2a-04bf-476d-bfeb-214fcb72d4db","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-wg4gd not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:34.266Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"0466815a-fb3c-4e06-815a-0ae13db4e0d9"}
{"level":"info","ts":"2026-06-05T09:49:34.267Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"0466815a-fb3c-4e06-815a-0ae13db4e0d9"}
{"level":"info","ts":"2026-06-05T09:49:34.267Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"0466815a-fb3c-4e06-815a-0ae13db4e0d9","1 head service found":"long-running-wg4gd-head-svc"}
{"level":"info","ts":"2026-06-05T09:49:34.267Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"0466815a-fb3c-4e06-815a-0ae13db4e0d9","error":"head certificate secret ray-head-secret-long-running-wg4gd not found"}
{"level":"error","ts":"2026-06-05T09:49:34.267Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"0466815a-fb3c-4e06-815a-0ae13db4e0d9","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-wg4gd not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:34.267Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"0466815a-fb3c-4e06-815a-0ae13db4e0d9","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-wg4gd not found"}
{"level":"info","ts":"2026-06-05T09:49:34.267Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"0466815a-fb3c-4e06-815a-0ae13db4e0d9","filter labels":["test-ns-thrrw",{"ray.io/cluster":"long-running-wg4gd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:49:34.267Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"0466815a-fb3c-4e06-815a-0ae13db4e0d9","filter labels":["test-ns-thrrw",{"ray.io/cluster":"long-running-wg4gd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:49:34.267Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"0466815a-fb3c-4e06-815a-0ae13db4e0d9","filter labels":["test-ns-thrrw",{"ray.io/cluster":"long-running-wg4gd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:49:34.267Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"0466815a-fb3c-4e06-815a-0ae13db4e0d9"}
{"level":"error","ts":"2026-06-05T09:49:34.267Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"0466815a-fb3c-4e06-815a-0ae13db4e0d9","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-wg4gd not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:34.587Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"74d21d4e-ae22-4b20-8b59-f19a804f5b8b"}
{"level":"info","ts":"2026-06-05T09:49:34.587Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"74d21d4e-ae22-4b20-8b59-f19a804f5b8b"}
{"level":"info","ts":"2026-06-05T09:49:34.587Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"74d21d4e-ae22-4b20-8b59-f19a804f5b8b","1 head service found":"long-running-wg4gd-head-svc"}
{"level":"info","ts":"2026-06-05T09:49:34.587Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"74d21d4e-ae22-4b20-8b59-f19a804f5b8b","error":"head certificate secret ray-head-secret-long-running-wg4gd not found"}
{"level":"error","ts":"2026-06-05T09:49:34.587Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"74d21d4e-ae22-4b20-8b59-f19a804f5b8b","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-wg4gd not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:34.587Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"74d21d4e-ae22-4b20-8b59-f19a804f5b8b","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-wg4gd not found"}
{"level":"info","ts":"2026-06-05T09:49:34.587Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"74d21d4e-ae22-4b20-8b59-f19a804f5b8b","filter labels":["test-ns-thrrw",{"ray.io/cluster":"long-running-wg4gd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:49:34.587Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"74d21d4e-ae22-4b20-8b59-f19a804f5b8b","filter labels":["test-ns-thrrw",{"ray.io/cluster":"long-running-wg4gd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:49:34.587Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"74d21d4e-ae22-4b20-8b59-f19a804f5b8b","filter labels":["test-ns-thrrw",{"ray.io/cluster":"long-running-wg4gd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:49:34.587Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"74d21d4e-ae22-4b20-8b59-f19a804f5b8b"}
{"level":"error","ts":"2026-06-05T09:49:34.587Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"74d21d4e-ae22-4b20-8b59-f19a804f5b8b","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-wg4gd not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:35.228Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"b8a41ef5-33c2-4d44-a9e7-3669cc059eeb"}
{"level":"info","ts":"2026-06-05T09:49:35.228Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"b8a41ef5-33c2-4d44-a9e7-3669cc059eeb"}
{"level":"info","ts":"2026-06-05T09:49:35.228Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"b8a41ef5-33c2-4d44-a9e7-3669cc059eeb","1 head service found":"long-running-wg4gd-head-svc"}
{"level":"info","ts":"2026-06-05T09:49:35.228Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"b8a41ef5-33c2-4d44-a9e7-3669cc059eeb","error":"head certificate secret ray-head-secret-long-running-wg4gd not found"}
{"level":"error","ts":"2026-06-05T09:49:35.228Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"b8a41ef5-33c2-4d44-a9e7-3669cc059eeb","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-wg4gd not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:35.228Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"b8a41ef5-33c2-4d44-a9e7-3669cc059eeb","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-wg4gd not found"}
{"level":"info","ts":"2026-06-05T09:49:35.228Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"b8a41ef5-33c2-4d44-a9e7-3669cc059eeb","filter labels":["test-ns-thrrw",{"ray.io/cluster":"long-running-wg4gd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:49:35.228Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"b8a41ef5-33c2-4d44-a9e7-3669cc059eeb","filter labels":["test-ns-thrrw",{"ray.io/cluster":"long-running-wg4gd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:49:35.228Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"b8a41ef5-33c2-4d44-a9e7-3669cc059eeb","filter labels":["test-ns-thrrw",{"ray.io/cluster":"long-running-wg4gd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:49:35.228Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"b8a41ef5-33c2-4d44-a9e7-3669cc059eeb"}
{"level":"error","ts":"2026-06-05T09:49:35.228Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"b8a41ef5-33c2-4d44-a9e7-3669cc059eeb","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-wg4gd not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:36.509Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7b16c112-06bc-4d2a-866e-cbbe0a8b5d22"}
{"level":"info","ts":"2026-06-05T09:49:36.509Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7b16c112-06bc-4d2a-866e-cbbe0a8b5d22"}
{"level":"info","ts":"2026-06-05T09:49:36.509Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7b16c112-06bc-4d2a-866e-cbbe0a8b5d22","1 head service found":"long-running-wg4gd-head-svc"}
{"level":"info","ts":"2026-06-05T09:49:36.509Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7b16c112-06bc-4d2a-866e-cbbe0a8b5d22"}
{"level":"info","ts":"2026-06-05T09:49:36.509Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7b16c112-06bc-4d2a-866e-cbbe0a8b5d22","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:49:36.509Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7b16c112-06bc-4d2a-866e-cbbe0a8b5d22"}
{"level":"info","ts":"2026-06-05T09:49:36.509Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7b16c112-06bc-4d2a-866e-cbbe0a8b5d22","mode":"IntegratedOAuth","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:36.509Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7b16c112-06bc-4d2a-866e-cbbe0a8b5d22","cluster":"long-running-wg4gd","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:49:36.509Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7b16c112-06bc-4d2a-866e-cbbe0a8b5d22","cluster":"long-running-wg4gd","authType":"IntegratedOAuth","serviceAccount":"long-running-wg4gd-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-05T09:49:36.509Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7b16c112-06bc-4d2a-866e-cbbe0a8b5d22","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"long-running-wg4gd","ray.io/group":"headgroup","ray.io/identifier":"long-running-wg4gd-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-05T09:49:36.509Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7b16c112-06bc-4d2a-866e-cbbe0a8b5d22","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"long-running-wg4gd-head-svc.test-ns-thrrw.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-05T09:49:36.509Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7b16c112-06bc-4d2a-866e-cbbe0a8b5d22","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-wg4gd-head-svc.test-ns-thrrw.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-05T09:49:36.509Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7b16c112-06bc-4d2a-866e-cbbe0a8b5d22","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-wg4gd-head-svc.test-ns-thrrw.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-05T09:49:36.509Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7b16c112-06bc-4d2a-866e-cbbe0a8b5d22","enabled":true}
{"level":"info","ts":"2026-06-05T09:49:36.540Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7b16c112-06bc-4d2a-866e-cbbe0a8b5d22","name":"long-running-wg4gd-head-jwf4t"}
{"level":"info","ts":"2026-06-05T09:49:36.540Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7b16c112-06bc-4d2a-866e-cbbe0a8b5d22","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:49:36.540Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7b16c112-06bc-4d2a-866e-cbbe0a8b5d22","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:36.540Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7b16c112-06bc-4d2a-866e-cbbe0a8b5d22","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-05T09:49:36.540Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7b16c112-06bc-4d2a-866e-cbbe0a8b5d22","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:36.540Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7b16c112-06bc-4d2a-866e-cbbe0a8b5d22","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-05T09:49:36.540Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7b16c112-06bc-4d2a-866e-cbbe0a8b5d22"}
{"level":"info","ts":"2026-06-05T09:49:36.540Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7b16c112-06bc-4d2a-866e-cbbe0a8b5d22","nodeType":"worker","rayStartParams":{"address":"long-running-wg4gd-head-svc.test-ns-thrrw.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-05T09:49:36.540Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7b16c112-06bc-4d2a-866e-cbbe0a8b5d22","rayStartCmd":"ray start  --address=long-running-wg4gd-head-svc.test-ns-thrrw.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:49:36.540Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7b16c112-06bc-4d2a-866e-cbbe0a8b5d22","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-wg4gd-head-svc.test-ns-thrrw.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:49:36.540Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7b16c112-06bc-4d2a-866e-cbbe0a8b5d22","enabled":true}
{"level":"info","ts":"2026-06-05T09:49:36.557Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"ca4e8d26-49ff-48b6-861e-b28bd4933d07","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:49:36.557Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"ca4e8d26-49ff-48b6-861e-b28bd4933d07","pod":"raycluster-head-vjd7d","ip":"10.134.0.20"}
{"level":"info","ts":"2026-06-05T09:49:36.557Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"ca4e8d26-49ff-48b6-861e-b28bd4933d07","pod":"raycluster-small-group-worker-j9pbw","ip":"10.133.0.35"}
{"level":"info","ts":"2026-06-05T09:49:36.557Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"},"namespace":"test-ns-rfwvh","name":"raycluster","reconcileID":"ca4e8d26-49ff-48b6-861e-b28bd4933d07","rayCluster":"raycluster","podIPs":["10.134.0.20","10.133.0.35"]}
{"level":"info","ts":"2026-06-05T09:49:36.580Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7b16c112-06bc-4d2a-866e-cbbe0a8b5d22","name":"long-running-wg4gd-small-group-worker-p2rz8"}
{"level":"info","ts":"2026-06-05T09:49:36.580Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7b16c112-06bc-4d2a-866e-cbbe0a8b5d22","podName":"long-running-wg4gd-head-jwf4t","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:49:36.580Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7b16c112-06bc-4d2a-866e-cbbe0a8b5d22","podName":"long-running-wg4gd-head-jwf4t","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:49:36.580Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7b16c112-06bc-4d2a-866e-cbbe0a8b5d22","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"long-running-wg4gd-head-svc"},"newHead":{"podName":"long-running-wg4gd-head-jwf4t","serviceName":"long-running-wg4gd-head-svc"}}
{"level":"info","ts":"2026-06-05T09:49:36.580Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7b16c112-06bc-4d2a-866e-cbbe0a8b5d22","name":"long-running-wg4gd","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:49:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-wg4gd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:49:33Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-wg4gd-head-jwf4t","serviceName":"long-running-wg4gd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:49:33Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:49:36.599Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"fc6cb7c7-3d3e-470d-a534-74a2d7288d0e","rayCluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:36.599Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"5c43a042-4869-4e7f-a974-f0736b32063d","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:36.599Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"b11e8029-70dd-4f71-83c4-015eb0b4bc2a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:36.599Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"b11e8029-70dd-4f71-83c4-015eb0b4bc2a","name":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:36.599Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"b11e8029-70dd-4f71-83c4-015eb0b4bc2a","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:49:36.599Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"fc6cb7c7-3d3e-470d-a534-74a2d7288d0e","rayCluster":"long-running-wg4gd","podIPs":[]}
{"level":"info","ts":"2026-06-05T09:49:36.599Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"b11e8029-70dd-4f71-83c4-015eb0b4bc2a"}
{"level":"info","ts":"2026-06-05T09:49:36.599Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"b11e8029-70dd-4f71-83c4-015eb0b4bc2a","RayCluster":"long-running-wg4gd","State":""}
{"level":"info","ts":"2026-06-05T09:49:36.600Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"1d118a50-b558-4ca0-877d-cce2e3ce49e0"}
{"level":"info","ts":"2026-06-05T09:49:36.600Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"1d118a50-b558-4ca0-877d-cce2e3ce49e0"}
{"level":"info","ts":"2026-06-05T09:49:36.600Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"1d118a50-b558-4ca0-877d-cce2e3ce49e0","1 head service found":"long-running-wg4gd-head-svc"}
{"level":"info","ts":"2026-06-05T09:49:36.600Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"1d118a50-b558-4ca0-877d-cce2e3ce49e0","Found 1 head Pod":"long-running-wg4gd-head-jwf4t","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:49:36.600Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"1d118a50-b558-4ca0-877d-cce2e3ce49e0","head Pod":"long-running-wg4gd-head-jwf4t","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-wg4gd-head-jwf4t. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:49:36.600Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"1d118a50-b558-4ca0-877d-cce2e3ce49e0","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:49:36.600Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"1d118a50-b558-4ca0-877d-cce2e3ce49e0","worker Pod":"long-running-wg4gd-small-group-worker-p2rz8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-wg4gd-small-group-worker-p2rz8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:49:36.600Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"1d118a50-b558-4ca0-877d-cce2e3ce49e0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:36.600Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"1d118a50-b558-4ca0-877d-cce2e3ce49e0","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:49:36.600Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"1d118a50-b558-4ca0-877d-cce2e3ce49e0","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:36.600Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"1d118a50-b558-4ca0-877d-cce2e3ce49e0","podName":"long-running-wg4gd-head-jwf4t","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:49:36.600Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"1d118a50-b558-4ca0-877d-cce2e3ce49e0","podName":"long-running-wg4gd-head-jwf4t","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:49:36.601Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"1d118a50-b558-4ca0-877d-cce2e3ce49e0","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:49:36.601Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"1d118a50-b558-4ca0-877d-cce2e3ce49e0","seconds":300}
{"level":"info","ts":"2026-06-05T09:49:36.615Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c8036293-7a0f-450e-bc3a-56daa7ccfc6d"}
{"level":"info","ts":"2026-06-05T09:49:36.615Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c8036293-7a0f-450e-bc3a-56daa7ccfc6d"}
{"level":"info","ts":"2026-06-05T09:49:36.615Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c8036293-7a0f-450e-bc3a-56daa7ccfc6d","1 head service found":"long-running-wg4gd-head-svc"}
{"level":"info","ts":"2026-06-05T09:49:36.615Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c8036293-7a0f-450e-bc3a-56daa7ccfc6d","Found 1 head Pod":"long-running-wg4gd-head-jwf4t","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:49:36.615Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c8036293-7a0f-450e-bc3a-56daa7ccfc6d","head Pod":"long-running-wg4gd-head-jwf4t","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-wg4gd-head-jwf4t. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:49:36.615Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c8036293-7a0f-450e-bc3a-56daa7ccfc6d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:49:36.615Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c8036293-7a0f-450e-bc3a-56daa7ccfc6d","worker Pod":"long-running-wg4gd-small-group-worker-p2rz8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-wg4gd-small-group-worker-p2rz8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:49:36.615Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c8036293-7a0f-450e-bc3a-56daa7ccfc6d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:36.615Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c8036293-7a0f-450e-bc3a-56daa7ccfc6d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:49:36.615Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c8036293-7a0f-450e-bc3a-56daa7ccfc6d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:36.615Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c8036293-7a0f-450e-bc3a-56daa7ccfc6d","podName":"long-running-wg4gd-head-jwf4t","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:49:36.615Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c8036293-7a0f-450e-bc3a-56daa7ccfc6d","podName":"long-running-wg4gd-head-jwf4t","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:49:36.615Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c8036293-7a0f-450e-bc3a-56daa7ccfc6d","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:49:33Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:49:33Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-05T09:49:36.615Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c8036293-7a0f-450e-bc3a-56daa7ccfc6d","name":"long-running-wg4gd","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-wg4gd-head-jwf4t","serviceName":"long-running-wg4gd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:49:33Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-wg4gd-head-jwf4t","serviceName":"long-running-wg4gd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:49:33Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:49:36.625Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"5c43a042-4869-4e7f-a974-f0736b32063d","name":"long-running-wg4gd-head"}
{"level":"info","ts":"2026-06-05T09:49:36.632Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"3e2ab3b4-0d66-4159-8799-e07867e563c3","rayCluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:36.632Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"8e801c6f-347b-4073-be5e-3110ae7b38da","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:36.632Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"8e801c6f-347b-4073-be5e-3110ae7b38da","name":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:36.632Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"8e801c6f-347b-4073-be5e-3110ae7b38da","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:49:36.632Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"3e2ab3b4-0d66-4159-8799-e07867e563c3","rayCluster":"long-running-wg4gd","podIPs":[]}
{"level":"info","ts":"2026-06-05T09:49:36.633Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"8e801c6f-347b-4073-be5e-3110ae7b38da"}
{"level":"info","ts":"2026-06-05T09:49:36.633Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"8e801c6f-347b-4073-be5e-3110ae7b38da","RayCluster":"long-running-wg4gd","State":""}
{"level":"info","ts":"2026-06-05T09:49:36.633Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"75744adc-9739-4cfb-a8c9-81bfd83db226"}
{"level":"info","ts":"2026-06-05T09:49:36.633Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"75744adc-9739-4cfb-a8c9-81bfd83db226"}
{"level":"info","ts":"2026-06-05T09:49:36.633Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"75744adc-9739-4cfb-a8c9-81bfd83db226","1 head service found":"long-running-wg4gd-head-svc"}
{"level":"info","ts":"2026-06-05T09:49:36.633Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"75744adc-9739-4cfb-a8c9-81bfd83db226","Found 1 head Pod":"long-running-wg4gd-head-jwf4t","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:49:36.633Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"75744adc-9739-4cfb-a8c9-81bfd83db226","head Pod":"long-running-wg4gd-head-jwf4t","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-wg4gd-head-jwf4t. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:49:36.633Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"75744adc-9739-4cfb-a8c9-81bfd83db226","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:49:36.633Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"75744adc-9739-4cfb-a8c9-81bfd83db226","worker Pod":"long-running-wg4gd-small-group-worker-p2rz8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-wg4gd-small-group-worker-p2rz8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:49:36.633Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"75744adc-9739-4cfb-a8c9-81bfd83db226","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:36.633Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"75744adc-9739-4cfb-a8c9-81bfd83db226","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:49:36.633Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"75744adc-9739-4cfb-a8c9-81bfd83db226","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:36.633Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"75744adc-9739-4cfb-a8c9-81bfd83db226","podName":"long-running-wg4gd-head-jwf4t","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:49:36.633Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"75744adc-9739-4cfb-a8c9-81bfd83db226","podName":"long-running-wg4gd-head-jwf4t","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:49:36.633Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"75744adc-9739-4cfb-a8c9-81bfd83db226","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:49:36.633Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"75744adc-9739-4cfb-a8c9-81bfd83db226","seconds":300}
{"level":"info","ts":"2026-06-05T09:49:36.648Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"5c43a042-4869-4e7f-a974-f0736b32063d","name":"long-running-wg4gd-workers"}
{"level":"info","ts":"2026-06-05T09:49:36.648Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"5c43a042-4869-4e7f-a974-f0736b32063d","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:36.648Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"e5d8f642-b23a-40f8-a34f-f6dd518a198d","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:36.648Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"daee6bee-cddb-478c-91f0-8f8b114a9b2d"}
{"level":"info","ts":"2026-06-05T09:49:36.649Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"daee6bee-cddb-478c-91f0-8f8b114a9b2d"}
{"level":"info","ts":"2026-06-05T09:49:36.649Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"daee6bee-cddb-478c-91f0-8f8b114a9b2d","1 head service found":"long-running-wg4gd-head-svc"}
{"level":"info","ts":"2026-06-05T09:49:36.649Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"daee6bee-cddb-478c-91f0-8f8b114a9b2d","Found 1 head Pod":"long-running-wg4gd-head-jwf4t","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:49:36.649Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"daee6bee-cddb-478c-91f0-8f8b114a9b2d","head Pod":"long-running-wg4gd-head-jwf4t","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-wg4gd-head-jwf4t. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:49:36.649Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"daee6bee-cddb-478c-91f0-8f8b114a9b2d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:49:36.649Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"daee6bee-cddb-478c-91f0-8f8b114a9b2d","worker Pod":"long-running-wg4gd-small-group-worker-p2rz8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-wg4gd-small-group-worker-p2rz8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:49:36.649Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"daee6bee-cddb-478c-91f0-8f8b114a9b2d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:36.649Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"daee6bee-cddb-478c-91f0-8f8b114a9b2d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:49:36.649Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"daee6bee-cddb-478c-91f0-8f8b114a9b2d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:36.649Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"daee6bee-cddb-478c-91f0-8f8b114a9b2d","podName":"long-running-wg4gd-head-jwf4t","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:49:36.649Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"daee6bee-cddb-478c-91f0-8f8b114a9b2d","podName":"long-running-wg4gd-head-jwf4t","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:49:36.649Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"daee6bee-cddb-478c-91f0-8f8b114a9b2d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:49:36.649Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"daee6bee-cddb-478c-91f0-8f8b114a9b2d","seconds":300}
{"level":"info","ts":"2026-06-05T09:49:36.711Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"e5d8f642-b23a-40f8-a34f-f6dd518a198d","name":"long-running-wg4gd-head"}
{"level":"info","ts":"2026-06-05T09:49:36.722Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"e5d8f642-b23a-40f8-a34f-f6dd518a198d","name":"long-running-wg4gd-workers"}
{"level":"info","ts":"2026-06-05T09:49:36.722Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"e5d8f642-b23a-40f8-a34f-f6dd518a198d","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:36.797Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"5319cfcd-400e-44fa-a89d-5dd264bb6394","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:36.797Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"5319cfcd-400e-44fa-a89d-5dd264bb6394","name":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:36.797Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"5319cfcd-400e-44fa-a89d-5dd264bb6394","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:49:36.797Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"5319cfcd-400e-44fa-a89d-5dd264bb6394"}
{"level":"info","ts":"2026-06-05T09:49:36.797Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"5319cfcd-400e-44fa-a89d-5dd264bb6394","RayCluster":"long-running-wg4gd","State":""}
{"level":"info","ts":"2026-06-05T09:49:36.997Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"77f8f804-93eb-4a33-a013-77b49677d551"}
{"level":"info","ts":"2026-06-05T09:49:36.997Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"77f8f804-93eb-4a33-a013-77b49677d551"}
{"level":"info","ts":"2026-06-05T09:49:36.997Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"77f8f804-93eb-4a33-a013-77b49677d551","1 head service found":"long-running-wg4gd-head-svc"}
{"level":"info","ts":"2026-06-05T09:49:36.997Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"77f8f804-93eb-4a33-a013-77b49677d551","Found 1 head Pod":"long-running-wg4gd-head-jwf4t","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:49:36.997Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"77f8f804-93eb-4a33-a013-77b49677d551","head Pod":"long-running-wg4gd-head-jwf4t","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-wg4gd-head-jwf4t. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:49:36.997Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"77f8f804-93eb-4a33-a013-77b49677d551","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:49:36.998Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"77f8f804-93eb-4a33-a013-77b49677d551","worker Pod":"long-running-wg4gd-small-group-worker-p2rz8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-wg4gd-small-group-worker-p2rz8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:49:36.998Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"77f8f804-93eb-4a33-a013-77b49677d551","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:36.998Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"77f8f804-93eb-4a33-a013-77b49677d551","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:49:36.998Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"77f8f804-93eb-4a33-a013-77b49677d551","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:36.998Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"77f8f804-93eb-4a33-a013-77b49677d551","podName":"long-running-wg4gd-head-jwf4t","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:49:36.998Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"77f8f804-93eb-4a33-a013-77b49677d551","podName":"long-running-wg4gd-head-jwf4t","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:49:36.998Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"77f8f804-93eb-4a33-a013-77b49677d551","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:49:36.998Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"77f8f804-93eb-4a33-a013-77b49677d551","seconds":300}
{"level":"info","ts":"2026-06-05T09:49:37.062Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c9e8c030-2de0-42b4-8876-b0f485ace2b4"}
{"level":"info","ts":"2026-06-05T09:49:37.062Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c9e8c030-2de0-42b4-8876-b0f485ace2b4"}
{"level":"info","ts":"2026-06-05T09:49:37.062Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c9e8c030-2de0-42b4-8876-b0f485ace2b4","1 head service found":"long-running-wg4gd-head-svc"}
{"level":"info","ts":"2026-06-05T09:49:37.062Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c9e8c030-2de0-42b4-8876-b0f485ace2b4","Found 1 head Pod":"long-running-wg4gd-head-jwf4t","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:49:37.062Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c9e8c030-2de0-42b4-8876-b0f485ace2b4","head Pod":"long-running-wg4gd-head-jwf4t","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-wg4gd-head-jwf4t. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:49:37.062Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c9e8c030-2de0-42b4-8876-b0f485ace2b4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:49:37.062Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c9e8c030-2de0-42b4-8876-b0f485ace2b4","worker Pod":"long-running-wg4gd-small-group-worker-p2rz8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-wg4gd-small-group-worker-p2rz8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:49:37.062Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c9e8c030-2de0-42b4-8876-b0f485ace2b4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:37.062Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c9e8c030-2de0-42b4-8876-b0f485ace2b4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:49:37.062Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c9e8c030-2de0-42b4-8876-b0f485ace2b4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:37.062Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c9e8c030-2de0-42b4-8876-b0f485ace2b4","podName":"long-running-wg4gd-head-jwf4t","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:49:37.062Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c9e8c030-2de0-42b4-8876-b0f485ace2b4","podName":"long-running-wg4gd-head-jwf4t","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:49:37.063Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c9e8c030-2de0-42b4-8876-b0f485ace2b4","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:49:37.063Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c9e8c030-2de0-42b4-8876-b0f485ace2b4","seconds":300}
{"level":"info","ts":"2026-06-05T09:49:37.246Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"120b99e8-184b-4bf2-815c-69da19342cb2"}
{"level":"info","ts":"2026-06-05T09:49:37.246Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"120b99e8-184b-4bf2-815c-69da19342cb2"}
{"level":"info","ts":"2026-06-05T09:49:37.246Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"120b99e8-184b-4bf2-815c-69da19342cb2","1 head service found":"long-running-wg4gd-head-svc"}
{"level":"info","ts":"2026-06-05T09:49:37.246Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"120b99e8-184b-4bf2-815c-69da19342cb2","Found 1 head Pod":"long-running-wg4gd-head-jwf4t","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:49:37.246Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"120b99e8-184b-4bf2-815c-69da19342cb2","head Pod":"long-running-wg4gd-head-jwf4t","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-wg4gd-head-jwf4t. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:49:37.246Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"120b99e8-184b-4bf2-815c-69da19342cb2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:49:37.246Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"120b99e8-184b-4bf2-815c-69da19342cb2","worker Pod":"long-running-wg4gd-small-group-worker-p2rz8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-wg4gd-small-group-worker-p2rz8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:49:37.246Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"120b99e8-184b-4bf2-815c-69da19342cb2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:37.246Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"120b99e8-184b-4bf2-815c-69da19342cb2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:49:37.246Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"120b99e8-184b-4bf2-815c-69da19342cb2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:37.247Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"120b99e8-184b-4bf2-815c-69da19342cb2","podName":"long-running-wg4gd-head-jwf4t","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:49:37.247Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"120b99e8-184b-4bf2-815c-69da19342cb2","podName":"long-running-wg4gd-head-jwf4t","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:49:37.247Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"120b99e8-184b-4bf2-815c-69da19342cb2","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:49:37.247Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"120b99e8-184b-4bf2-815c-69da19342cb2","seconds":300}
{"level":"info","ts":"2026-06-05T09:49:37.953Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"89927aa3-1141-4500-aeed-32fb969c8f3b"}
{"level":"info","ts":"2026-06-05T09:49:37.953Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"89927aa3-1141-4500-aeed-32fb969c8f3b"}
{"level":"info","ts":"2026-06-05T09:49:37.953Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"89927aa3-1141-4500-aeed-32fb969c8f3b","1 head service found":"long-running-wg4gd-head-svc"}
{"level":"info","ts":"2026-06-05T09:49:37.953Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"89927aa3-1141-4500-aeed-32fb969c8f3b","Found 1 head Pod":"long-running-wg4gd-head-jwf4t","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:49:37.953Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"89927aa3-1141-4500-aeed-32fb969c8f3b","head Pod":"long-running-wg4gd-head-jwf4t","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-wg4gd-head-jwf4t. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:49:37.953Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"89927aa3-1141-4500-aeed-32fb969c8f3b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:49:37.953Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"89927aa3-1141-4500-aeed-32fb969c8f3b","worker Pod":"long-running-wg4gd-small-group-worker-p2rz8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-wg4gd-small-group-worker-p2rz8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:49:37.953Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"89927aa3-1141-4500-aeed-32fb969c8f3b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:37.953Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"89927aa3-1141-4500-aeed-32fb969c8f3b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:49:37.953Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"89927aa3-1141-4500-aeed-32fb969c8f3b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:37.953Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"89927aa3-1141-4500-aeed-32fb969c8f3b","podName":"long-running-wg4gd-head-jwf4t","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:49:37.953Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"89927aa3-1141-4500-aeed-32fb969c8f3b","podName":"long-running-wg4gd-head-jwf4t","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:49:37.953Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"89927aa3-1141-4500-aeed-32fb969c8f3b","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"long-running-wg4gd-head-jwf4t","serviceName":"long-running-wg4gd-head-svc"},"newHead":{"podIP":"10.132.0.26","serviceIP":"10.132.0.26","podName":"long-running-wg4gd-head-jwf4t","serviceName":"long-running-wg4gd-head-svc"}}
{"level":"info","ts":"2026-06-05T09:49:37.953Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"89927aa3-1141-4500-aeed-32fb969c8f3b","name":"long-running-wg4gd","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:49:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-wg4gd-head-jwf4t","serviceName":"long-running-wg4gd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:49:33Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:49:37Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.26","serviceIP":"10.132.0.26","podName":"long-running-wg4gd-head-jwf4t","serviceName":"long-running-wg4gd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:49:33Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:49:37.966Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"98fa979c-4d71-4d4e-8e49-ee5f715785c9","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:37.966Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"b5ec66ad-d2eb-4ba8-9ad6-6d4f875e7463","rayCluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:37.966Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"4a762621-bd69-428f-87c3-4aebe9493d1f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:37.966Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"4a762621-bd69-428f-87c3-4aebe9493d1f","name":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:37.966Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"4a762621-bd69-428f-87c3-4aebe9493d1f","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:49:37.966Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"b5ec66ad-d2eb-4ba8-9ad6-6d4f875e7463","pod":"long-running-wg4gd-head-jwf4t","ip":"10.132.0.26"}
{"level":"info","ts":"2026-06-05T09:49:37.966Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"b5ec66ad-d2eb-4ba8-9ad6-6d4f875e7463","pod":"long-running-wg4gd-small-group-worker-p2rz8","ip":"10.133.0.53"}
{"level":"info","ts":"2026-06-05T09:49:37.966Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"4a762621-bd69-428f-87c3-4aebe9493d1f"}
{"level":"info","ts":"2026-06-05T09:49:37.966Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"4a762621-bd69-428f-87c3-4aebe9493d1f","RayCluster":"long-running-wg4gd","State":""}
{"level":"info","ts":"2026-06-05T09:49:37.980Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"98fa979c-4d71-4d4e-8e49-ee5f715785c9","name":"long-running-wg4gd-head"}
{"level":"info","ts":"2026-06-05T09:49:37.981Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"b5ec66ad-d2eb-4ba8-9ad6-6d4f875e7463","certificate":"ray-head-cert-long-running-wg4gd","dnsNames":["long-running-wg4gd-head-svc","localhost","long-running-wg4gd-head-svc.test-ns-thrrw.svc","long-running-wg4gd-head-svc.test-ns-thrrw.svc.cluster.local"],"ipAddresses":["10.132.0.26","10.133.0.53","127.0.0.1"]}
{"level":"info","ts":"2026-06-05T09:49:37.992Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"98fa979c-4d71-4d4e-8e49-ee5f715785c9","name":"long-running-wg4gd-workers"}
{"level":"info","ts":"2026-06-05T09:49:37.992Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"98fa979c-4d71-4d4e-8e49-ee5f715785c9","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:37.993Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"b5ec66ad-d2eb-4ba8-9ad6-6d4f875e7463","certificate":"ray-worker-cert-long-running-wg4gd","dnsNames":["long-running-wg4gd-worker-svc","localhost","long-running-wg4gd-small-group","long-running-wg4gd-worker-svc.test-ns-thrrw.svc","long-running-wg4gd-worker-svc.test-ns-thrrw.svc.cluster.local","*.long-running-wg4gd-worker-svc.test-ns-thrrw.svc","*.long-running-wg4gd-worker-svc.test-ns-thrrw.svc.cluster.local","*-worker-*.test-ns-thrrw.svc","*-worker-*.test-ns-thrrw.svc.cluster.local"],"ipAddresses":["10.132.0.26","10.133.0.53","127.0.0.1"]}
{"level":"info","ts":"2026-06-05T09:49:37.993Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"b5ec66ad-d2eb-4ba8-9ad6-6d4f875e7463","rayCluster":"long-running-wg4gd","podIPs":["10.132.0.26","10.133.0.53"]}
{"level":"info","ts":"2026-06-05T09:49:38.601Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"27735395-c60f-43ed-8046-f38e816d7afe"}
{"level":"info","ts":"2026-06-05T09:49:38.601Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"27735395-c60f-43ed-8046-f38e816d7afe"}
{"level":"info","ts":"2026-06-05T09:49:38.601Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"27735395-c60f-43ed-8046-f38e816d7afe","1 head service found":"long-running-wg4gd-head-svc"}
{"level":"info","ts":"2026-06-05T09:49:38.601Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"27735395-c60f-43ed-8046-f38e816d7afe","Found 1 head Pod":"long-running-wg4gd-head-jwf4t","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:49:38.601Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"27735395-c60f-43ed-8046-f38e816d7afe","head Pod":"long-running-wg4gd-head-jwf4t","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-wg4gd-head-jwf4t. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:49:38.601Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"27735395-c60f-43ed-8046-f38e816d7afe","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:49:38.601Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"27735395-c60f-43ed-8046-f38e816d7afe","worker Pod":"long-running-wg4gd-small-group-worker-p2rz8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-wg4gd-small-group-worker-p2rz8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:49:38.601Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"27735395-c60f-43ed-8046-f38e816d7afe","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:38.601Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"27735395-c60f-43ed-8046-f38e816d7afe","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:49:38.601Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"27735395-c60f-43ed-8046-f38e816d7afe","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:38.601Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"27735395-c60f-43ed-8046-f38e816d7afe","podName":"long-running-wg4gd-head-jwf4t","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:49:38.601Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"27735395-c60f-43ed-8046-f38e816d7afe","podName":"long-running-wg4gd-head-jwf4t","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:49:38.601Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"27735395-c60f-43ed-8046-f38e816d7afe","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:49:38.601Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"27735395-c60f-43ed-8046-f38e816d7afe","seconds":300}
{"level":"error","ts":"2026-06-05T09:49:39.027Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-thrrw"},"reconcileID":"6fe770e3-f6e8-4bd2-b416-3dbbf40e10f7","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:39.027Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-thrrw"},"reconcileID":"6fe770e3-f6e8-4bd2-b416-3dbbf40e10f7"}
{"level":"error","ts":"2026-06-05T09:49:39.027Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-thrrw"},"reconcileID":"6fe770e3-f6e8-4bd2-b416-3dbbf40e10f7","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:39.798Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"ccd16b84-7bfe-45e2-bc87-cc9846b57bf8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:39.798Z","logger":"controllers.RayJob","msg":"The RayJob has passed the activeDeadlineSeconds. Transition the status to `Failed`.","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"ccd16b84-7bfe-45e2-bc87-cc9846b57bf8","StartTime":"2026-06-05 09:49:33 +0000 UTC","ActiveDeadlineSeconds":5}
{"level":"info","ts":"2026-06-05T09:49:39.798Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"ccd16b84-7bfe-45e2-bc87-cc9846b57bf8","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-77jkz","rayClusterName":"long-running-wg4gd","jobDeploymentStatus":"Initializing","startTime":"2026-06-05T09:49:33Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-77jkz","rayClusterName":"long-running-wg4gd","jobDeploymentStatus":"Failed","reason":"DeadlineExceeded","message":"The RayJob has passed the activeDeadlineSeconds. StartTime: 2026-06-05 09:49:33 +0000 UTC. ActiveDeadlineSeconds: 5","startTime":"2026-06-05T09:49:33Z","succeeded":0,"failed":1,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:49:39.798Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"ccd16b84-7bfe-45e2-bc87-cc9846b57bf8","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Failed"}
{"level":"info","ts":"2026-06-05T09:49:39.810Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"164a4a03-8a7b-4a71-b9b7-9176bcc7dece","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:39.811Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"164a4a03-8a7b-4a71-b9b7-9176bcc7dece","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-06-05 09:49:39 +0000 UTC","Now":"2026-06-05T09:49:39.811Z","ShutdownTime":"2026-06-05T09:59:39.000Z"}
{"level":"info","ts":"2026-06-05T09:49:39.811Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"164a4a03-8a7b-4a71-b9b7-9176bcc7dece","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-05T09:49:39.811Z","logger":"controllers.RayJob","msg":"shutdownTime not reached, requeue this RayJob for n seconds","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"164a4a03-8a7b-4a71-b9b7-9176bcc7dece","seconds":601}
{"level":"info","ts":"2026-06-05T09:49:40.796Z","logger":"controllers.RayJob","msg":"Skipping RayJob managed by a custom controller","RayJob":{"name":"managed-externally","namespace":"test-ns-thrrw"},"reconcileID":"cdd3fa31-49ae-4459-993d-b676b94e9b90","managed-by":"kueue.x-k8s.io/multikueue"}
{"level":"info","ts":"2026-06-05T09:49:40.829Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"managed-externally","namespace":"test-ns-thrrw"},"reconcileID":"38e3eaf5-39db-483b-833c-db9fe0c82e36"}
{"level":"info","ts":"2026-06-05T09:49:40.911Z","logger":"controllers.RayJob","msg":"Skipping RayJob managed by a custom controller","RayJob":{"name":"managed-externally","namespace":"test-ns-rfwvh"},"reconcileID":"bc505bea-84f1-49a6-8c63-8df620625510","managed-by":"kueue.x-k8s.io/multikueue"}
{"level":"info","ts":"2026-06-05T09:49:42.811Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"f043e3f3-88cc-4ab7-8d8d-bbef1eac8eca","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:42.811Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"f043e3f3-88cc-4ab7-8d8d-bbef1eac8eca","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-06-05 09:49:39 +0000 UTC","Now":"2026-06-05T09:49:42.811Z","ShutdownTime":"2026-06-05T09:59:39.000Z"}
{"level":"info","ts":"2026-06-05T09:49:42.811Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"f043e3f3-88cc-4ab7-8d8d-bbef1eac8eca","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-05T09:49:42.811Z","logger":"controllers.RayJob","msg":"shutdownTime not reached, requeue this RayJob for n seconds","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"f043e3f3-88cc-4ab7-8d8d-bbef1eac8eca","seconds":598}
{"level":"info","ts":"2026-06-05T09:49:42.972Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"2acdfc26-42a5-4349-b9b6-30c093fac8e8"}
{"level":"info","ts":"2026-06-05T09:49:42.972Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"2acdfc26-42a5-4349-b9b6-30c093fac8e8"}
{"level":"info","ts":"2026-06-05T09:49:42.972Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"2acdfc26-42a5-4349-b9b6-30c093fac8e8","1 head service found":"long-running-wg4gd-head-svc"}
{"level":"info","ts":"2026-06-05T09:49:42.972Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"2acdfc26-42a5-4349-b9b6-30c093fac8e8","Found 1 head Pod":"long-running-wg4gd-head-jwf4t","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:49:42.972Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"2acdfc26-42a5-4349-b9b6-30c093fac8e8","head Pod":"long-running-wg4gd-head-jwf4t","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-wg4gd-head-jwf4t. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:49:42.972Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"2acdfc26-42a5-4349-b9b6-30c093fac8e8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:49:42.972Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"2acdfc26-42a5-4349-b9b6-30c093fac8e8","worker Pod":"long-running-wg4gd-small-group-worker-p2rz8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-wg4gd-small-group-worker-p2rz8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:49:42.972Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"2acdfc26-42a5-4349-b9b6-30c093fac8e8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:42.972Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"2acdfc26-42a5-4349-b9b6-30c093fac8e8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:49:42.972Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"2acdfc26-42a5-4349-b9b6-30c093fac8e8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:42.972Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"2acdfc26-42a5-4349-b9b6-30c093fac8e8","podName":"long-running-wg4gd-head-jwf4t","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:49:42.972Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"2acdfc26-42a5-4349-b9b6-30c093fac8e8","podName":"long-running-wg4gd-head-jwf4t","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-05T09:49:42.972Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"2acdfc26-42a5-4349-b9b6-30c093fac8e8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:49:42.972Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"2acdfc26-42a5-4349-b9b6-30c093fac8e8","seconds":300}
{"level":"info","ts":"2026-06-05T09:49:43.925Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"efe9760c-128b-4f0b-9f93-9fc0a7c806d5","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:43.925Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"efe9760c-128b-4f0b-9f93-9fc0a7c806d5","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-05T09:49:43.935Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"efe9760c-128b-4f0b-9f93-9fc0a7c806d5"}
{"level":"info","ts":"2026-06-05T09:49:43.935Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"efe9760c-128b-4f0b-9f93-9fc0a7c806d5","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-05T09:49:43.935Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"efe9760c-128b-4f0b-9f93-9fc0a7c806d5","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-65xg8","rayClusterName":"raycluster","jobDeploymentStatus":"Initializing","startTime":"2026-06-05T09:49:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:49:43.935Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"efe9760c-128b-4f0b-9f93-9fc0a7c806d5","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-05T09:49:43.945Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"7bd0adf6-c5c1-4449-8c6a-82a4ce53ebcf","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:43.945Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"7bd0adf6-c5c1-4449-8c6a-82a4ce53ebcf","name":"raycluster"}
{"level":"info","ts":"2026-06-05T09:49:43.945Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"7bd0adf6-c5c1-4449-8c6a-82a4ce53ebcf","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"}}
{"level":"info","ts":"2026-06-05T09:49:43.945Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"7bd0adf6-c5c1-4449-8c6a-82a4ce53ebcf","head service name":"raycluster-head-svc","namespace":"test-ns-rfwvh"}
{"level":"info","ts":"2026-06-05T09:49:43.945Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"7bd0adf6-c5c1-4449-8c6a-82a4ce53ebcf","head service URL":"raycluster-head-svc.test-ns-rfwvh.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-05T09:49:43.945Z","logger":"controllers.RayJob","msg":"user-provided submitter template is used; the first container is assumed to be the submitter","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"7bd0adf6-c5c1-4449-8c6a-82a4ce53ebcf"}
{"level":"info","ts":"2026-06-05T09:49:43.945Z","logger":"controllers.RayJob","msg":"No command is specified in the user-provided template. Default command is used","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"7bd0adf6-c5c1-4449-8c6a-82a4ce53ebcf","command":["if","!","ray","job","status","--address","http://raycluster-head-svc.test-ns-rfwvh.svc.cluster.local:8265","counter-65xg8",">/dev/null","2>&1",";","then","ray","job","submit","--address","http://raycluster-head-svc.test-ns-rfwvh.svc.cluster.local:8265","--no-wait","--runtime-env-json","\"{\\\"env_vars\\\":{\\\"counter_name\\\":\\\"test_counter\\\"}}\"","--submission-id","counter-65xg8","--","python /home/ray/jobs/counter.py",";","fi",";","ray","job","logs","--address","http://raycluster-head-svc.test-ns-rfwvh.svc.cluster.local:8265","--follow","counter-65xg8"]}
{"level":"info","ts":"2026-06-05T09:49:43.957Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"7bd0adf6-c5c1-4449-8c6a-82a4ce53ebcf","Kubernetes Job":"counter"}
{"level":"info","ts":"2026-06-05T09:49:43.957Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"7bd0adf6-c5c1-4449-8c6a-82a4ce53ebcf","SubmissionMode":"K8sJobMode","RayCluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:49:43.957Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"7bd0adf6-c5c1-4449-8c6a-82a4ce53ebcf","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-65xg8","rayClusterName":"raycluster","jobDeploymentStatus":"Initializing","startTime":"2026-06-05T09:49:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-65xg8","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-rfwvh.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-05T09:49:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:49:43.957Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"7bd0adf6-c5c1-4449-8c6a-82a4ce53ebcf","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-05T09:49:43.967Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"b3054e20-7a02-4474-bc8a-c2177a635d87","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:43.968Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"b3054e20-7a02-4474-bc8a-c2177a635d87","name":"raycluster"}
{"level":"info","ts":"2026-06-05T09:49:43.968Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"b3054e20-7a02-4474-bc8a-c2177a635d87","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"}}
{"level":"info","ts":"2026-06-05T09:49:43.978Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"3944ea43-22a7-4e5b-b7e6-21fcb90e0592"}
{"level":"info","ts":"2026-06-05T09:49:43.978Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"3944ea43-22a7-4e5b-b7e6-21fcb90e0592"}
{"level":"info","ts":"2026-06-05T09:49:43.978Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"3944ea43-22a7-4e5b-b7e6-21fcb90e0592","1 head service found":"long-running-wg4gd-head-svc"}
{"level":"info","ts":"2026-06-05T09:49:43.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"3944ea43-22a7-4e5b-b7e6-21fcb90e0592","Found 1 head Pod":"long-running-wg4gd-head-jwf4t","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:49:43.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"3944ea43-22a7-4e5b-b7e6-21fcb90e0592","head Pod":"long-running-wg4gd-head-jwf4t","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-wg4gd-head-jwf4t. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:49:43.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"3944ea43-22a7-4e5b-b7e6-21fcb90e0592","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:49:43.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"3944ea43-22a7-4e5b-b7e6-21fcb90e0592","worker Pod":"long-running-wg4gd-small-group-worker-p2rz8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-wg4gd-small-group-worker-p2rz8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:49:43.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"3944ea43-22a7-4e5b-b7e6-21fcb90e0592","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:43.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"3944ea43-22a7-4e5b-b7e6-21fcb90e0592","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:49:43.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"3944ea43-22a7-4e5b-b7e6-21fcb90e0592","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:43.978Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"3944ea43-22a7-4e5b-b7e6-21fcb90e0592","podName":"long-running-wg4gd-head-jwf4t","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:49:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:49:43.978Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"3944ea43-22a7-4e5b-b7e6-21fcb90e0592","podName":"long-running-wg4gd-head-jwf4t","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:49:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:49:43.978Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"3944ea43-22a7-4e5b-b7e6-21fcb90e0592","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:49:33Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:49:33Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-05T09:49:43.978Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"3944ea43-22a7-4e5b-b7e6-21fcb90e0592","name":"long-running-wg4gd","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:49:37Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.26","serviceIP":"10.132.0.26","podName":"long-running-wg4gd-head-jwf4t","serviceName":"long-running-wg4gd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:49:33Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:49:43Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.26","serviceIP":"10.132.0.26","podName":"long-running-wg4gd-head-jwf4t","serviceName":"long-running-wg4gd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:49:33Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"error","ts":"2026-06-05T09:49:43.979Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"b3054e20-7a02-4474-bc8a-c2177a635d87","JobId":"counter-65xg8","error":"Job counter-65xg8 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:43.979Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"b3054e20-7a02-4474-bc8a-c2177a635d87"}
{"level":"error","ts":"2026-06-05T09:49:43.979Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"b3054e20-7a02-4474-bc8a-c2177a635d87","error":"Job counter-65xg8 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:43.979Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"f8a3f77c-86f1-48fb-b6a8-35db1978216f","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:43.979Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"f8a3f77c-86f1-48fb-b6a8-35db1978216f","name":"raycluster"}
{"level":"info","ts":"2026-06-05T09:49:43.979Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"f8a3f77c-86f1-48fb-b6a8-35db1978216f","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"}}
{"level":"info","ts":"2026-06-05T09:49:44.006Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"e3144109-e116-4594-90c7-86b3f4933414","rayCluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:44.007Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"e3144109-e116-4594-90c7-86b3f4933414","pod":"long-running-wg4gd-small-group-worker-p2rz8","ip":"10.133.0.53"}
{"level":"info","ts":"2026-06-05T09:49:44.007Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"e3144109-e116-4594-90c7-86b3f4933414","pod":"long-running-wg4gd-head-jwf4t","ip":"10.132.0.26"}
{"level":"info","ts":"2026-06-05T09:49:44.007Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"de98aa62-f185-4ea2-9572-e82c4d3650ec","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:44.007Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"e3144109-e116-4594-90c7-86b3f4933414","rayCluster":"long-running-wg4gd","podIPs":["10.133.0.53","10.132.0.26"]}
{"level":"error","ts":"2026-06-05T09:49:44.010Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"f8a3f77c-86f1-48fb-b6a8-35db1978216f","JobId":"counter-65xg8","error":"Job counter-65xg8 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:44.011Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"f8a3f77c-86f1-48fb-b6a8-35db1978216f"}
{"level":"error","ts":"2026-06-05T09:49:44.011Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"f8a3f77c-86f1-48fb-b6a8-35db1978216f","error":"Job counter-65xg8 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:44.011Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"01ca98c4-94ab-4840-b4df-fe7ce6a1b4fd","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:44.011Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"01ca98c4-94ab-4840-b4df-fe7ce6a1b4fd","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-06-05 09:49:39 +0000 UTC","Now":"2026-06-05T09:49:44.011Z","ShutdownTime":"2026-06-05T09:59:39.000Z"}
{"level":"info","ts":"2026-06-05T09:49:44.011Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"01ca98c4-94ab-4840-b4df-fe7ce6a1b4fd","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-05T09:49:44.011Z","logger":"controllers.RayJob","msg":"shutdownTime not reached, requeue this RayJob for n seconds","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"01ca98c4-94ab-4840-b4df-fe7ce6a1b4fd","seconds":596}
{"level":"info","ts":"2026-06-05T09:49:44.011Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"b10b0655-6f2b-4052-95d8-0afc428c1dcc","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:44.011Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"b10b0655-6f2b-4052-95d8-0afc428c1dcc","name":"raycluster"}
{"level":"info","ts":"2026-06-05T09:49:44.011Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"b10b0655-6f2b-4052-95d8-0afc428c1dcc","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"}}
{"level":"info","ts":"2026-06-05T09:49:44.013Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"de98aa62-f185-4ea2-9572-e82c4d3650ec","cluster":"long-running-wg4gd"}
{"level":"error","ts":"2026-06-05T09:49:44.015Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"b10b0655-6f2b-4052-95d8-0afc428c1dcc","JobId":"counter-65xg8","error":"Job counter-65xg8 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:44.015Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"b10b0655-6f2b-4052-95d8-0afc428c1dcc"}
{"level":"error","ts":"2026-06-05T09:49:44.015Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"b10b0655-6f2b-4052-95d8-0afc428c1dcc","error":"Job counter-65xg8 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:44.021Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"1a621b5d-8f9e-4f1c-b8d7-582dc569333c","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:44.021Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"1a621b5d-8f9e-4f1c-b8d7-582dc569333c","name":"raycluster"}
{"level":"info","ts":"2026-06-05T09:49:44.021Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"1a621b5d-8f9e-4f1c-b8d7-582dc569333c","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"}}
{"level":"error","ts":"2026-06-05T09:49:44.110Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"1a621b5d-8f9e-4f1c-b8d7-582dc569333c","JobId":"counter-65xg8","error":"Job counter-65xg8 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:44.110Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"1a621b5d-8f9e-4f1c-b8d7-582dc569333c"}
{"level":"error","ts":"2026-06-05T09:49:44.110Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"1a621b5d-8f9e-4f1c-b8d7-582dc569333c","error":"Job counter-65xg8 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:44.150Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"a6bab476-75d0-48b7-a3f6-a98a7d463330","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:44.150Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"a6bab476-75d0-48b7-a3f6-a98a7d463330","name":"raycluster"}
{"level":"info","ts":"2026-06-05T09:49:44.150Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"a6bab476-75d0-48b7-a3f6-a98a7d463330","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"}}
{"level":"error","ts":"2026-06-05T09:49:44.155Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"a6bab476-75d0-48b7-a3f6-a98a7d463330","JobId":"counter-65xg8","error":"Job counter-65xg8 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:44.156Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"a6bab476-75d0-48b7-a3f6-a98a7d463330"}
{"level":"error","ts":"2026-06-05T09:49:44.156Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"a6bab476-75d0-48b7-a3f6-a98a7d463330","error":"Job counter-65xg8 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:44.236Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"22f088ba-7020-41aa-a7e6-ac1f263082eb","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:44.236Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"22f088ba-7020-41aa-a7e6-ac1f263082eb","name":"raycluster"}
{"level":"info","ts":"2026-06-05T09:49:44.236Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"22f088ba-7020-41aa-a7e6-ac1f263082eb","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"}}
{"level":"error","ts":"2026-06-05T09:49:44.241Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"22f088ba-7020-41aa-a7e6-ac1f263082eb","JobId":"counter-65xg8","error":"Job counter-65xg8 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:44.241Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"22f088ba-7020-41aa-a7e6-ac1f263082eb"}
{"level":"error","ts":"2026-06-05T09:49:44.241Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"22f088ba-7020-41aa-a7e6-ac1f263082eb","error":"Job counter-65xg8 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:44.402Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"e8a41d97-5221-4ac5-a525-c7672e45a40f","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:44.402Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"e8a41d97-5221-4ac5-a525-c7672e45a40f","name":"raycluster"}
{"level":"info","ts":"2026-06-05T09:49:44.402Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"e8a41d97-5221-4ac5-a525-c7672e45a40f","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"}}
{"level":"error","ts":"2026-06-05T09:49:44.409Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"e8a41d97-5221-4ac5-a525-c7672e45a40f","JobId":"counter-65xg8","error":"Job counter-65xg8 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:44.409Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"e8a41d97-5221-4ac5-a525-c7672e45a40f"}
{"level":"error","ts":"2026-06-05T09:49:44.409Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"e8a41d97-5221-4ac5-a525-c7672e45a40f","error":"Job counter-65xg8 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:44.730Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"a67ee9dd-ee75-4481-8ad0-2d88b2b5f148","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:44.730Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"a67ee9dd-ee75-4481-8ad0-2d88b2b5f148","name":"raycluster"}
{"level":"info","ts":"2026-06-05T09:49:44.731Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"a67ee9dd-ee75-4481-8ad0-2d88b2b5f148","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"}}
{"level":"error","ts":"2026-06-05T09:49:44.736Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"a67ee9dd-ee75-4481-8ad0-2d88b2b5f148","JobId":"counter-65xg8","error":"Job counter-65xg8 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:44.736Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"a67ee9dd-ee75-4481-8ad0-2d88b2b5f148"}
{"level":"error","ts":"2026-06-05T09:49:44.736Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"a67ee9dd-ee75-4481-8ad0-2d88b2b5f148","error":"Job counter-65xg8 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:45.377Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"24e7f6ac-0bf6-4bfe-a1d7-e8f8dab363c2","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:45.378Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"24e7f6ac-0bf6-4bfe-a1d7-e8f8dab363c2","name":"raycluster"}
{"level":"info","ts":"2026-06-05T09:49:45.378Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"24e7f6ac-0bf6-4bfe-a1d7-e8f8dab363c2","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"}}
{"level":"error","ts":"2026-06-05T09:49:45.383Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"24e7f6ac-0bf6-4bfe-a1d7-e8f8dab363c2","JobId":"counter-65xg8","error":"Job counter-65xg8 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:45.383Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"24e7f6ac-0bf6-4bfe-a1d7-e8f8dab363c2"}
{"level":"error","ts":"2026-06-05T09:49:45.383Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"24e7f6ac-0bf6-4bfe-a1d7-e8f8dab363c2","error":"Job counter-65xg8 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:45.925Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"309eba64-1d8f-4993-80c8-c119bffecf3a"}
{"level":"info","ts":"2026-06-05T09:49:45.925Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"309eba64-1d8f-4993-80c8-c119bffecf3a"}
{"level":"info","ts":"2026-06-05T09:49:45.925Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"309eba64-1d8f-4993-80c8-c119bffecf3a","1 head service found":"long-running-wg4gd-head-svc"}
{"level":"info","ts":"2026-06-05T09:49:45.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"309eba64-1d8f-4993-80c8-c119bffecf3a","Found 1 head Pod":"long-running-wg4gd-head-jwf4t","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:49:45.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"309eba64-1d8f-4993-80c8-c119bffecf3a","head Pod":"long-running-wg4gd-head-jwf4t","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-wg4gd-head-jwf4t. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:49:45.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"309eba64-1d8f-4993-80c8-c119bffecf3a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:49:45.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"309eba64-1d8f-4993-80c8-c119bffecf3a","worker Pod":"long-running-wg4gd-small-group-worker-p2rz8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-wg4gd-small-group-worker-p2rz8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:49:45.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"309eba64-1d8f-4993-80c8-c119bffecf3a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:45.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"309eba64-1d8f-4993-80c8-c119bffecf3a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:49:45.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"309eba64-1d8f-4993-80c8-c119bffecf3a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:45.925Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"309eba64-1d8f-4993-80c8-c119bffecf3a","podName":"long-running-wg4gd-head-jwf4t","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:49:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:49:45.925Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"309eba64-1d8f-4993-80c8-c119bffecf3a","podName":"long-running-wg4gd-head-jwf4t","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:49:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:49:45.926Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"309eba64-1d8f-4993-80c8-c119bffecf3a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:49:45.926Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"309eba64-1d8f-4993-80c8-c119bffecf3a","seconds":300}
{"level":"info","ts":"2026-06-05T09:49:45.930Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"5a55a3df-acd4-4cd3-9f8c-e8d3623e0561"}
{"level":"info","ts":"2026-06-05T09:49:45.930Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"5a55a3df-acd4-4cd3-9f8c-e8d3623e0561"}
{"level":"info","ts":"2026-06-05T09:49:45.930Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"5a55a3df-acd4-4cd3-9f8c-e8d3623e0561","1 head service found":"long-running-wg4gd-head-svc"}
{"level":"info","ts":"2026-06-05T09:49:45.930Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"5a55a3df-acd4-4cd3-9f8c-e8d3623e0561","Found 1 head Pod":"long-running-wg4gd-head-jwf4t","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:49:45.930Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"5a55a3df-acd4-4cd3-9f8c-e8d3623e0561","head Pod":"long-running-wg4gd-head-jwf4t","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-wg4gd-head-jwf4t. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:49:45.930Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"5a55a3df-acd4-4cd3-9f8c-e8d3623e0561","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:49:45.931Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"5a55a3df-acd4-4cd3-9f8c-e8d3623e0561","worker Pod":"long-running-wg4gd-small-group-worker-p2rz8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-wg4gd-small-group-worker-p2rz8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:49:45.931Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"5a55a3df-acd4-4cd3-9f8c-e8d3623e0561","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:45.931Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"5a55a3df-acd4-4cd3-9f8c-e8d3623e0561","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:49:45.931Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"5a55a3df-acd4-4cd3-9f8c-e8d3623e0561","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:45.931Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"5a55a3df-acd4-4cd3-9f8c-e8d3623e0561","podName":"long-running-wg4gd-head-jwf4t","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:49:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:49:45.931Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"5a55a3df-acd4-4cd3-9f8c-e8d3623e0561","podName":"long-running-wg4gd-head-jwf4t","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:49:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:49:45.931Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"5a55a3df-acd4-4cd3-9f8c-e8d3623e0561","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:49:45.931Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"5a55a3df-acd4-4cd3-9f8c-e8d3623e0561","seconds":300}
{"level":"info","ts":"2026-06-05T09:49:45.951Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"a30d9006-62c0-42a2-815b-b143fd3cc72c"}
{"level":"info","ts":"2026-06-05T09:49:45.951Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"a30d9006-62c0-42a2-815b-b143fd3cc72c"}
{"level":"info","ts":"2026-06-05T09:49:45.951Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"a30d9006-62c0-42a2-815b-b143fd3cc72c","1 head service found":"long-running-wg4gd-head-svc"}
{"level":"info","ts":"2026-06-05T09:49:45.951Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"a30d9006-62c0-42a2-815b-b143fd3cc72c","Found 1 head Pod":"long-running-wg4gd-head-jwf4t","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:49:45.951Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"a30d9006-62c0-42a2-815b-b143fd3cc72c","head Pod":"long-running-wg4gd-head-jwf4t","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-wg4gd-head-jwf4t. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:49:45.951Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"a30d9006-62c0-42a2-815b-b143fd3cc72c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:49:45.951Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"a30d9006-62c0-42a2-815b-b143fd3cc72c","worker Pod":"long-running-wg4gd-small-group-worker-p2rz8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-wg4gd-small-group-worker-p2rz8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:49:45.951Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"a30d9006-62c0-42a2-815b-b143fd3cc72c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:45.951Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"a30d9006-62c0-42a2-815b-b143fd3cc72c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:49:45.951Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"a30d9006-62c0-42a2-815b-b143fd3cc72c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:45.951Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"a30d9006-62c0-42a2-815b-b143fd3cc72c","podName":"long-running-wg4gd-head-jwf4t","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:49:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:49:45.951Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"a30d9006-62c0-42a2-815b-b143fd3cc72c","podName":"long-running-wg4gd-head-jwf4t","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:49:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:49:45.951Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"a30d9006-62c0-42a2-815b-b143fd3cc72c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:49:45.951Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"a30d9006-62c0-42a2-815b-b143fd3cc72c","seconds":300}
{"level":"info","ts":"2026-06-05T09:49:45.984Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c7858064-859a-4abe-9f8f-2e4cf260547a"}
{"level":"info","ts":"2026-06-05T09:49:45.984Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c7858064-859a-4abe-9f8f-2e4cf260547a"}
{"level":"info","ts":"2026-06-05T09:49:45.984Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c7858064-859a-4abe-9f8f-2e4cf260547a","1 head service found":"long-running-wg4gd-head-svc"}
{"level":"info","ts":"2026-06-05T09:49:45.984Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c7858064-859a-4abe-9f8f-2e4cf260547a","Found 1 head Pod":"long-running-wg4gd-head-jwf4t","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:49:45.984Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c7858064-859a-4abe-9f8f-2e4cf260547a","head Pod":"long-running-wg4gd-head-jwf4t","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-wg4gd-head-jwf4t. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:49:45.984Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c7858064-859a-4abe-9f8f-2e4cf260547a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:49:45.984Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c7858064-859a-4abe-9f8f-2e4cf260547a","worker Pod":"long-running-wg4gd-small-group-worker-p2rz8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-wg4gd-small-group-worker-p2rz8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:49:45.984Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c7858064-859a-4abe-9f8f-2e4cf260547a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:45.984Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c7858064-859a-4abe-9f8f-2e4cf260547a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:49:45.984Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c7858064-859a-4abe-9f8f-2e4cf260547a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:45.984Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c7858064-859a-4abe-9f8f-2e4cf260547a","podName":"long-running-wg4gd-head-jwf4t","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:49:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:49:45.984Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c7858064-859a-4abe-9f8f-2e4cf260547a","podName":"long-running-wg4gd-head-jwf4t","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:49:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:49:45.984Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c7858064-859a-4abe-9f8f-2e4cf260547a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:49:45.984Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c7858064-859a-4abe-9f8f-2e4cf260547a","seconds":300}
{"level":"info","ts":"2026-06-05T09:49:46.007Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"3185ad34-3957-4984-9866-5c790ba64600"}
{"level":"info","ts":"2026-06-05T09:49:46.007Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"3185ad34-3957-4984-9866-5c790ba64600"}
{"level":"info","ts":"2026-06-05T09:49:46.007Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"3185ad34-3957-4984-9866-5c790ba64600","1 head service found":"long-running-wg4gd-head-svc"}
{"level":"info","ts":"2026-06-05T09:49:46.007Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"3185ad34-3957-4984-9866-5c790ba64600","Found 1 head Pod":"long-running-wg4gd-head-jwf4t","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:49:46.007Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"3185ad34-3957-4984-9866-5c790ba64600","head Pod":"long-running-wg4gd-head-jwf4t","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-wg4gd-head-jwf4t. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:49:46.007Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"3185ad34-3957-4984-9866-5c790ba64600","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:49:46.007Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"3185ad34-3957-4984-9866-5c790ba64600","worker Pod":"long-running-wg4gd-small-group-worker-p2rz8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-wg4gd-small-group-worker-p2rz8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:49:46.007Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"3185ad34-3957-4984-9866-5c790ba64600","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:46.007Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"3185ad34-3957-4984-9866-5c790ba64600","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:49:46.007Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"3185ad34-3957-4984-9866-5c790ba64600","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:46.007Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"3185ad34-3957-4984-9866-5c790ba64600","podName":"long-running-wg4gd-head-jwf4t","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:49:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:49:46.007Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"3185ad34-3957-4984-9866-5c790ba64600","podName":"long-running-wg4gd-head-jwf4t","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:49:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:49:46.007Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"3185ad34-3957-4984-9866-5c790ba64600","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:49:46.007Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"3185ad34-3957-4984-9866-5c790ba64600","seconds":300}
{"level":"info","ts":"2026-06-05T09:49:46.292Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"00d33fdb-483d-4984-844f-1d35a38ca66e","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:46.292Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"00d33fdb-483d-4984-844f-1d35a38ca66e","name":"raycluster"}
{"level":"info","ts":"2026-06-05T09:49:46.292Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"00d33fdb-483d-4984-844f-1d35a38ca66e","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"}}
{"level":"error","ts":"2026-06-05T09:49:46.297Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"00d33fdb-483d-4984-844f-1d35a38ca66e","JobId":"counter-65xg8","error":"Job counter-65xg8 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:46.298Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"00d33fdb-483d-4984-844f-1d35a38ca66e"}
{"level":"error","ts":"2026-06-05T09:49:46.298Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"00d33fdb-483d-4984-844f-1d35a38ca66e","error":"Job counter-65xg8 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:46.664Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"e28ec2b7-9e6b-4431-bd4b-ed1e45924e68","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:46.664Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"e28ec2b7-9e6b-4431-bd4b-ed1e45924e68","name":"raycluster"}
{"level":"info","ts":"2026-06-05T09:49:46.664Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"e28ec2b7-9e6b-4431-bd4b-ed1e45924e68","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"}}
{"level":"error","ts":"2026-06-05T09:49:46.670Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"e28ec2b7-9e6b-4431-bd4b-ed1e45924e68","JobId":"counter-65xg8","error":"Job counter-65xg8 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:46.670Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"e28ec2b7-9e6b-4431-bd4b-ed1e45924e68"}
{"level":"error","ts":"2026-06-05T09:49:46.670Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"e28ec2b7-9e6b-4431-bd4b-ed1e45924e68","error":"Job counter-65xg8 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:47.016Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"ca094887-1913-4e94-9591-21d823339ec6"}
{"level":"info","ts":"2026-06-05T09:49:47.016Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"ca094887-1913-4e94-9591-21d823339ec6"}
{"level":"info","ts":"2026-06-05T09:49:47.016Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"ca094887-1913-4e94-9591-21d823339ec6","1 head service found":"long-running-wg4gd-head-svc"}
{"level":"info","ts":"2026-06-05T09:49:47.016Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"ca094887-1913-4e94-9591-21d823339ec6","Found 1 head Pod":"long-running-wg4gd-head-jwf4t","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:49:47.016Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"ca094887-1913-4e94-9591-21d823339ec6","head Pod":"long-running-wg4gd-head-jwf4t","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-wg4gd-head-jwf4t. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:49:47.016Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"ca094887-1913-4e94-9591-21d823339ec6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:49:47.016Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"ca094887-1913-4e94-9591-21d823339ec6","worker Pod":"long-running-wg4gd-small-group-worker-p2rz8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-wg4gd-small-group-worker-p2rz8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:49:47.016Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"ca094887-1913-4e94-9591-21d823339ec6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:47.016Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"ca094887-1913-4e94-9591-21d823339ec6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-05T09:49:47.016Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"ca094887-1913-4e94-9591-21d823339ec6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:47.016Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"ca094887-1913-4e94-9591-21d823339ec6","podName":"long-running-wg4gd-head-jwf4t","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:49:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:49:47.016Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"ca094887-1913-4e94-9591-21d823339ec6","podName":"long-running-wg4gd-head-jwf4t","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:49:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:49:47.017Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"ca094887-1913-4e94-9591-21d823339ec6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:49:47.017Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"ca094887-1913-4e94-9591-21d823339ec6","seconds":300}
{"level":"info","ts":"2026-06-05T09:49:47.312Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7c83a26a-a4cc-456f-836d-f71d4efb06ee"}
{"level":"info","ts":"2026-06-05T09:49:47.312Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7c83a26a-a4cc-456f-836d-f71d4efb06ee"}
{"level":"info","ts":"2026-06-05T09:49:47.312Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7c83a26a-a4cc-456f-836d-f71d4efb06ee","1 head service found":"long-running-wg4gd-head-svc"}
{"level":"info","ts":"2026-06-05T09:49:47.312Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7c83a26a-a4cc-456f-836d-f71d4efb06ee","Found 1 head Pod":"long-running-wg4gd-head-jwf4t","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:49:47.312Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7c83a26a-a4cc-456f-836d-f71d4efb06ee","head Pod":"long-running-wg4gd-head-jwf4t","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-wg4gd-head-jwf4t. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:49:47.312Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7c83a26a-a4cc-456f-836d-f71d4efb06ee","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:49:47.312Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7c83a26a-a4cc-456f-836d-f71d4efb06ee","worker Pod":"long-running-wg4gd-small-group-worker-p2rz8","shouldDelete":true,"reason":"The worker Pod long-running-wg4gd-small-group-worker-p2rz8 status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"error","ts":"2026-06-05T09:49:47.315Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7c83a26a-a4cc-456f-836d-f71d4efb06ee","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"delete 1 unhealthy worker Pods","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:47.315Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7c83a26a-a4cc-456f-836d-f71d4efb06ee","error":"delete 1 unhealthy worker Pods"}
{"level":"info","ts":"2026-06-05T09:49:47.315Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7c83a26a-a4cc-456f-836d-f71d4efb06ee","podName":"long-running-wg4gd-head-jwf4t","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:49:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:49:47.315Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7c83a26a-a4cc-456f-836d-f71d4efb06ee"}
{"level":"error","ts":"2026-06-05T09:49:47.316Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7c83a26a-a4cc-456f-836d-f71d4efb06ee","error":"delete 1 unhealthy worker Pods","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:47.319Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"3765796d-c419-486c-a924-c9229890d3d7"}
{"level":"info","ts":"2026-06-05T09:49:47.319Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"3765796d-c419-486c-a924-c9229890d3d7"}
{"level":"info","ts":"2026-06-05T09:49:47.320Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"3765796d-c419-486c-a924-c9229890d3d7","1 head service found":"long-running-wg4gd-head-svc"}
{"level":"info","ts":"2026-06-05T09:49:47.320Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"3765796d-c419-486c-a924-c9229890d3d7","Found 1 head Pod":"long-running-wg4gd-head-jwf4t","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:49:47.320Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"3765796d-c419-486c-a924-c9229890d3d7","head Pod":"long-running-wg4gd-head-jwf4t","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-wg4gd-head-jwf4t. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:49:47.320Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"3765796d-c419-486c-a924-c9229890d3d7","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-06-05T09:49:47.320Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"3765796d-c419-486c-a924-c9229890d3d7","podName":"long-running-wg4gd-head-jwf4t","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:49:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:49:47.320Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"3765796d-c419-486c-a924-c9229890d3d7","podName":"long-running-wg4gd-head-jwf4t","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:49:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:49:47.320Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"3765796d-c419-486c-a924-c9229890d3d7","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:49:47.320Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"3765796d-c419-486c-a924-c9229890d3d7","seconds":300}
{"level":"info","ts":"2026-06-05T09:49:47.322Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"acad3e04-77c0-40da-98eb-4677f96e7fa7"}
{"level":"info","ts":"2026-06-05T09:49:47.322Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"acad3e04-77c0-40da-98eb-4677f96e7fa7"}
{"level":"info","ts":"2026-06-05T09:49:47.322Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"acad3e04-77c0-40da-98eb-4677f96e7fa7","1 head service found":"long-running-wg4gd-head-svc"}
{"level":"info","ts":"2026-06-05T09:49:47.322Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"acad3e04-77c0-40da-98eb-4677f96e7fa7","Found 1 head Pod":"long-running-wg4gd-head-jwf4t","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:49:47.322Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"acad3e04-77c0-40da-98eb-4677f96e7fa7","head Pod":"long-running-wg4gd-head-jwf4t","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-wg4gd-head-jwf4t. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:49:47.322Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"acad3e04-77c0-40da-98eb-4677f96e7fa7","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-06-05T09:49:47.322Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"acad3e04-77c0-40da-98eb-4677f96e7fa7","podName":"long-running-wg4gd-head-jwf4t","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:49:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:49:47.322Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"acad3e04-77c0-40da-98eb-4677f96e7fa7","podName":"long-running-wg4gd-head-jwf4t","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:49:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:49:47.322Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"acad3e04-77c0-40da-98eb-4677f96e7fa7","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-05T09:49:47.322Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"acad3e04-77c0-40da-98eb-4677f96e7fa7","seconds":300}
{"level":"info","ts":"2026-06-05T09:49:47.323Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c89dc02e-0ae2-420e-a629-8396728e788f"}
{"level":"info","ts":"2026-06-05T09:49:47.323Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c89dc02e-0ae2-420e-a629-8396728e788f"}
{"level":"info","ts":"2026-06-05T09:49:47.323Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c89dc02e-0ae2-420e-a629-8396728e788f","1 head service found":"long-running-wg4gd-head-svc"}
{"level":"info","ts":"2026-06-05T09:49:47.323Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c89dc02e-0ae2-420e-a629-8396728e788f","Found 1 head Pod":"long-running-wg4gd-head-jwf4t","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:49:47.323Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c89dc02e-0ae2-420e-a629-8396728e788f","head Pod":"long-running-wg4gd-head-jwf4t","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-wg4gd-head-jwf4t. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:49:47.324Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c89dc02e-0ae2-420e-a629-8396728e788f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:49:47.324Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c89dc02e-0ae2-420e-a629-8396728e788f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:47.324Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c89dc02e-0ae2-420e-a629-8396728e788f","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-05T09:49:47.324Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c89dc02e-0ae2-420e-a629-8396728e788f","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:47.324Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c89dc02e-0ae2-420e-a629-8396728e788f","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-05T09:49:47.324Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c89dc02e-0ae2-420e-a629-8396728e788f"}
{"level":"info","ts":"2026-06-05T09:49:47.324Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c89dc02e-0ae2-420e-a629-8396728e788f","nodeType":"worker","rayStartParams":{"address":"long-running-wg4gd-head-svc.test-ns-thrrw.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-05T09:49:47.324Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c89dc02e-0ae2-420e-a629-8396728e788f","rayStartCmd":"ray start  --address=long-running-wg4gd-head-svc.test-ns-thrrw.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:49:47.324Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c89dc02e-0ae2-420e-a629-8396728e788f","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-wg4gd-head-svc.test-ns-thrrw.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:49:47.324Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c89dc02e-0ae2-420e-a629-8396728e788f","enabled":true}
{"level":"error","ts":"2026-06-05T09:49:47.327Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c89dc02e-0ae2-420e-a629-8396728e788f","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-wg4gd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:47.327Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c89dc02e-0ae2-420e-a629-8396728e788f","error":"FailedCreateWorkerPod\npods \"long-running-wg4gd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated"}
{"level":"info","ts":"2026-06-05T09:49:47.327Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c89dc02e-0ae2-420e-a629-8396728e788f","podName":"long-running-wg4gd-head-jwf4t","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:49:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:49:47.327Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c89dc02e-0ae2-420e-a629-8396728e788f","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:49:33Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:49:33Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-05T09:49:47Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"long-running-wg4gd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated"}]}
{"level":"info","ts":"2026-06-05T09:49:47.327Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c89dc02e-0ae2-420e-a629-8396728e788f","name":"long-running-wg4gd","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:49:43Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.26","serviceIP":"10.132.0.26","podName":"long-running-wg4gd-head-jwf4t","serviceName":"long-running-wg4gd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:49:33Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:49:47Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.26","serviceIP":"10.132.0.26","podName":"long-running-wg4gd-head-jwf4t","serviceName":"long-running-wg4gd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:49:33Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-05T09:49:47Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"long-running-wg4gd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:49:47.340Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c89dc02e-0ae2-420e-a629-8396728e788f"}
{"level":"error","ts":"2026-06-05T09:49:47.340Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"c89dc02e-0ae2-420e-a629-8396728e788f","error":"FailedCreateWorkerPod\npods \"long-running-wg4gd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:47.340Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"1b74129c-09ef-4146-a13a-b2f0755e3484","rayCluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:47.340Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"bc57acda-9a17-4fd6-9ed4-87c46c3dfe49","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:47.340Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"1b74129c-09ef-4146-a13a-b2f0755e3484","pod":"long-running-wg4gd-head-jwf4t","ip":"10.132.0.26"}
{"level":"info","ts":"2026-06-05T09:49:47.341Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"3f166db4-1446-4563-9c2d-525e80dc69c9","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:47.341Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"3f166db4-1446-4563-9c2d-525e80dc69c9","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-06-05 09:49:39 +0000 UTC","Now":"2026-06-05T09:49:47.341Z","ShutdownTime":"2026-06-05T09:59:39.000Z"}
{"level":"info","ts":"2026-06-05T09:49:47.341Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"3f166db4-1446-4563-9c2d-525e80dc69c9","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-05T09:49:47.341Z","logger":"controllers.RayJob","msg":"shutdownTime not reached, requeue this RayJob for n seconds","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"3f166db4-1446-4563-9c2d-525e80dc69c9","seconds":593}
{"level":"info","ts":"2026-06-05T09:49:47.346Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"bc57acda-9a17-4fd6-9ed4-87c46c3dfe49","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:47.405Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"1b74129c-09ef-4146-a13a-b2f0755e3484","certificate":"ray-head-cert-long-running-wg4gd","dnsNames":["long-running-wg4gd-head-svc","localhost","long-running-wg4gd-head-svc.test-ns-thrrw.svc","long-running-wg4gd-head-svc.test-ns-thrrw.svc.cluster.local"],"ipAddresses":["10.132.0.26","127.0.0.1"]}
{"level":"info","ts":"2026-06-05T09:49:47.406Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"a4dedbb7-cdb6-4fee-b50f-47267f9ce310"}
{"level":"info","ts":"2026-06-05T09:49:47.406Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"a4dedbb7-cdb6-4fee-b50f-47267f9ce310"}
{"level":"info","ts":"2026-06-05T09:49:47.406Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"a4dedbb7-cdb6-4fee-b50f-47267f9ce310","1 head service found":"long-running-wg4gd-head-svc"}
{"level":"info","ts":"2026-06-05T09:49:47.406Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"a4dedbb7-cdb6-4fee-b50f-47267f9ce310","Found 1 head Pod":"long-running-wg4gd-head-jwf4t","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:49:47.406Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"a4dedbb7-cdb6-4fee-b50f-47267f9ce310","head Pod":"long-running-wg4gd-head-jwf4t","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-wg4gd-head-jwf4t. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:49:47.406Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"a4dedbb7-cdb6-4fee-b50f-47267f9ce310","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:49:47.406Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"a4dedbb7-cdb6-4fee-b50f-47267f9ce310","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:47.406Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"a4dedbb7-cdb6-4fee-b50f-47267f9ce310","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-05T09:49:47.406Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"a4dedbb7-cdb6-4fee-b50f-47267f9ce310","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:47.406Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"a4dedbb7-cdb6-4fee-b50f-47267f9ce310","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-05T09:49:47.406Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"a4dedbb7-cdb6-4fee-b50f-47267f9ce310"}
{"level":"info","ts":"2026-06-05T09:49:47.406Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"a4dedbb7-cdb6-4fee-b50f-47267f9ce310","nodeType":"worker","rayStartParams":{"address":"long-running-wg4gd-head-svc.test-ns-thrrw.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-05T09:49:47.406Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"a4dedbb7-cdb6-4fee-b50f-47267f9ce310","rayStartCmd":"ray start  --address=long-running-wg4gd-head-svc.test-ns-thrrw.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:49:47.406Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"a4dedbb7-cdb6-4fee-b50f-47267f9ce310","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-wg4gd-head-svc.test-ns-thrrw.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:49:47.406Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"a4dedbb7-cdb6-4fee-b50f-47267f9ce310","enabled":true}
{"level":"error","ts":"2026-06-05T09:49:47.410Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"a4dedbb7-cdb6-4fee-b50f-47267f9ce310","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-wg4gd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:47.410Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"a4dedbb7-cdb6-4fee-b50f-47267f9ce310","error":"FailedCreateWorkerPod\npods \"long-running-wg4gd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated"}
{"level":"info","ts":"2026-06-05T09:49:47.410Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"a4dedbb7-cdb6-4fee-b50f-47267f9ce310","podName":"long-running-wg4gd-head-jwf4t","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:49:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:49:47.410Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"a4dedbb7-cdb6-4fee-b50f-47267f9ce310"}
{"level":"error","ts":"2026-06-05T09:49:47.410Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"a4dedbb7-cdb6-4fee-b50f-47267f9ce310","error":"FailedCreateWorkerPod\npods \"long-running-wg4gd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:47.417Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"1b74129c-09ef-4146-a13a-b2f0755e3484","certificate":"ray-worker-cert-long-running-wg4gd","dnsNames":["long-running-wg4gd-worker-svc","localhost","long-running-wg4gd-small-group","long-running-wg4gd-worker-svc.test-ns-thrrw.svc","long-running-wg4gd-worker-svc.test-ns-thrrw.svc.cluster.local","*.long-running-wg4gd-worker-svc.test-ns-thrrw.svc","*.long-running-wg4gd-worker-svc.test-ns-thrrw.svc.cluster.local","*-worker-*.test-ns-thrrw.svc","*-worker-*.test-ns-thrrw.svc.cluster.local"],"ipAddresses":["10.132.0.26","127.0.0.1"]}
{"level":"info","ts":"2026-06-05T09:49:47.417Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"1b74129c-09ef-4146-a13a-b2f0755e3484"}
{"level":"info","ts":"2026-06-05T09:49:47.421Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"9dbc481c-ad4c-4428-9964-d1bc672d29d8"}
{"level":"info","ts":"2026-06-05T09:49:47.421Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"9dbc481c-ad4c-4428-9964-d1bc672d29d8"}
{"level":"info","ts":"2026-06-05T09:49:47.421Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"9dbc481c-ad4c-4428-9964-d1bc672d29d8","1 head service found":"long-running-wg4gd-head-svc"}
{"level":"info","ts":"2026-06-05T09:49:47.421Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"9dbc481c-ad4c-4428-9964-d1bc672d29d8","Found 1 head Pod":"long-running-wg4gd-head-jwf4t","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:49:47.421Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"9dbc481c-ad4c-4428-9964-d1bc672d29d8","head Pod":"long-running-wg4gd-head-jwf4t","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-wg4gd-head-jwf4t. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:49:47.421Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"9dbc481c-ad4c-4428-9964-d1bc672d29d8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:49:47.421Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"9dbc481c-ad4c-4428-9964-d1bc672d29d8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:47.421Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"9dbc481c-ad4c-4428-9964-d1bc672d29d8","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-05T09:49:47.421Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"9dbc481c-ad4c-4428-9964-d1bc672d29d8","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:47.421Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"9dbc481c-ad4c-4428-9964-d1bc672d29d8","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-05T09:49:47.421Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"9dbc481c-ad4c-4428-9964-d1bc672d29d8"}
{"level":"info","ts":"2026-06-05T09:49:47.421Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"9dbc481c-ad4c-4428-9964-d1bc672d29d8","nodeType":"worker","rayStartParams":{"address":"long-running-wg4gd-head-svc.test-ns-thrrw.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-05T09:49:47.421Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"9dbc481c-ad4c-4428-9964-d1bc672d29d8","rayStartCmd":"ray start  --address=long-running-wg4gd-head-svc.test-ns-thrrw.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:49:47.421Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"9dbc481c-ad4c-4428-9964-d1bc672d29d8","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-wg4gd-head-svc.test-ns-thrrw.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:49:47.421Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"9dbc481c-ad4c-4428-9964-d1bc672d29d8","enabled":true}
{"level":"error","ts":"2026-06-05T09:49:47.424Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"9dbc481c-ad4c-4428-9964-d1bc672d29d8","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-wg4gd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:47.425Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"9dbc481c-ad4c-4428-9964-d1bc672d29d8","error":"FailedCreateWorkerPod\npods \"long-running-wg4gd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated"}
{"level":"info","ts":"2026-06-05T09:49:47.425Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"9dbc481c-ad4c-4428-9964-d1bc672d29d8","podName":"long-running-wg4gd-head-jwf4t","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:49:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:49:47.425Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"9dbc481c-ad4c-4428-9964-d1bc672d29d8"}
{"level":"error","ts":"2026-06-05T09:49:47.425Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"9dbc481c-ad4c-4428-9964-d1bc672d29d8","error":"FailedCreateWorkerPod\npods \"long-running-wg4gd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:47.445Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"be4ae015-9371-46f8-be24-886f97e75c9b"}
{"level":"info","ts":"2026-06-05T09:49:47.445Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"be4ae015-9371-46f8-be24-886f97e75c9b"}
{"level":"info","ts":"2026-06-05T09:49:47.445Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"be4ae015-9371-46f8-be24-886f97e75c9b","1 head service found":"long-running-wg4gd-head-svc"}
{"level":"info","ts":"2026-06-05T09:49:47.446Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"be4ae015-9371-46f8-be24-886f97e75c9b","Found 1 head Pod":"long-running-wg4gd-head-jwf4t","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:49:47.446Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"be4ae015-9371-46f8-be24-886f97e75c9b","head Pod":"long-running-wg4gd-head-jwf4t","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-wg4gd-head-jwf4t. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:49:47.446Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"be4ae015-9371-46f8-be24-886f97e75c9b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:49:47.446Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"be4ae015-9371-46f8-be24-886f97e75c9b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:47.446Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"be4ae015-9371-46f8-be24-886f97e75c9b","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-05T09:49:47.446Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"be4ae015-9371-46f8-be24-886f97e75c9b","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:47.446Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"be4ae015-9371-46f8-be24-886f97e75c9b","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-05T09:49:47.446Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"be4ae015-9371-46f8-be24-886f97e75c9b"}
{"level":"info","ts":"2026-06-05T09:49:47.446Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"be4ae015-9371-46f8-be24-886f97e75c9b","nodeType":"worker","rayStartParams":{"address":"long-running-wg4gd-head-svc.test-ns-thrrw.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-05T09:49:47.446Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"be4ae015-9371-46f8-be24-886f97e75c9b","rayStartCmd":"ray start  --address=long-running-wg4gd-head-svc.test-ns-thrrw.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:49:47.446Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"be4ae015-9371-46f8-be24-886f97e75c9b","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-wg4gd-head-svc.test-ns-thrrw.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:49:47.446Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"be4ae015-9371-46f8-be24-886f97e75c9b","enabled":true}
{"level":"error","ts":"2026-06-05T09:49:47.449Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"be4ae015-9371-46f8-be24-886f97e75c9b","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-wg4gd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:47.450Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"be4ae015-9371-46f8-be24-886f97e75c9b","error":"FailedCreateWorkerPod\npods \"long-running-wg4gd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated"}
{"level":"info","ts":"2026-06-05T09:49:47.450Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"be4ae015-9371-46f8-be24-886f97e75c9b","podName":"long-running-wg4gd-head-jwf4t","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:49:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:49:47.450Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"be4ae015-9371-46f8-be24-886f97e75c9b"}
{"level":"error","ts":"2026-06-05T09:49:47.450Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"be4ae015-9371-46f8-be24-886f97e75c9b","error":"FailedCreateWorkerPod\npods \"long-running-wg4gd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:47.505Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"00d578b3-369b-4f38-99b9-0a50d2ccf5b8"}
{"level":"info","ts":"2026-06-05T09:49:47.505Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"00d578b3-369b-4f38-99b9-0a50d2ccf5b8"}
{"level":"info","ts":"2026-06-05T09:49:47.506Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"00d578b3-369b-4f38-99b9-0a50d2ccf5b8","1 head service found":"long-running-wg4gd-head-svc"}
{"level":"info","ts":"2026-06-05T09:49:47.506Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"00d578b3-369b-4f38-99b9-0a50d2ccf5b8","Found 1 head Pod":"long-running-wg4gd-head-jwf4t","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:49:47.506Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"00d578b3-369b-4f38-99b9-0a50d2ccf5b8","head Pod":"long-running-wg4gd-head-jwf4t","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-wg4gd-head-jwf4t. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:49:47.506Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"00d578b3-369b-4f38-99b9-0a50d2ccf5b8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:49:47.506Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"00d578b3-369b-4f38-99b9-0a50d2ccf5b8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:47.506Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"00d578b3-369b-4f38-99b9-0a50d2ccf5b8","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-05T09:49:47.506Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"00d578b3-369b-4f38-99b9-0a50d2ccf5b8","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:47.506Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"00d578b3-369b-4f38-99b9-0a50d2ccf5b8","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-05T09:49:47.506Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"00d578b3-369b-4f38-99b9-0a50d2ccf5b8"}
{"level":"info","ts":"2026-06-05T09:49:47.506Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"00d578b3-369b-4f38-99b9-0a50d2ccf5b8","nodeType":"worker","rayStartParams":{"address":"long-running-wg4gd-head-svc.test-ns-thrrw.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-05T09:49:47.506Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"00d578b3-369b-4f38-99b9-0a50d2ccf5b8","rayStartCmd":"ray start  --address=long-running-wg4gd-head-svc.test-ns-thrrw.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:49:47.506Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"00d578b3-369b-4f38-99b9-0a50d2ccf5b8","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-wg4gd-head-svc.test-ns-thrrw.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:49:47.506Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"00d578b3-369b-4f38-99b9-0a50d2ccf5b8","enabled":true}
{"level":"error","ts":"2026-06-05T09:49:47.510Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"00d578b3-369b-4f38-99b9-0a50d2ccf5b8","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-wg4gd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:47.510Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"00d578b3-369b-4f38-99b9-0a50d2ccf5b8","error":"FailedCreateWorkerPod\npods \"long-running-wg4gd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated"}
{"level":"info","ts":"2026-06-05T09:49:47.510Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"00d578b3-369b-4f38-99b9-0a50d2ccf5b8","podName":"long-running-wg4gd-head-jwf4t","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:49:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:49:47.510Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"00d578b3-369b-4f38-99b9-0a50d2ccf5b8"}
{"level":"error","ts":"2026-06-05T09:49:47.510Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"00d578b3-369b-4f38-99b9-0a50d2ccf5b8","error":"FailedCreateWorkerPod\npods \"long-running-wg4gd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:47.606Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"0ff0b3e8-c9b9-4196-af83-e512d4ab30f5"}
{"level":"info","ts":"2026-06-05T09:49:47.606Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"0ff0b3e8-c9b9-4196-af83-e512d4ab30f5"}
{"level":"info","ts":"2026-06-05T09:49:47.606Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"0ff0b3e8-c9b9-4196-af83-e512d4ab30f5","1 head service found":"long-running-wg4gd-head-svc"}
{"level":"info","ts":"2026-06-05T09:49:47.606Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"0ff0b3e8-c9b9-4196-af83-e512d4ab30f5","Found 1 head Pod":"long-running-wg4gd-head-jwf4t","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:49:47.606Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"0ff0b3e8-c9b9-4196-af83-e512d4ab30f5","head Pod":"long-running-wg4gd-head-jwf4t","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-wg4gd-head-jwf4t. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:49:47.606Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"0ff0b3e8-c9b9-4196-af83-e512d4ab30f5","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:49:47.606Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"0ff0b3e8-c9b9-4196-af83-e512d4ab30f5","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:47.606Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"0ff0b3e8-c9b9-4196-af83-e512d4ab30f5","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-05T09:49:47.606Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"0ff0b3e8-c9b9-4196-af83-e512d4ab30f5","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:47.606Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"0ff0b3e8-c9b9-4196-af83-e512d4ab30f5","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-05T09:49:47.606Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"0ff0b3e8-c9b9-4196-af83-e512d4ab30f5"}
{"level":"info","ts":"2026-06-05T09:49:47.606Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"0ff0b3e8-c9b9-4196-af83-e512d4ab30f5","nodeType":"worker","rayStartParams":{"address":"long-running-wg4gd-head-svc.test-ns-thrrw.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-05T09:49:47.606Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"0ff0b3e8-c9b9-4196-af83-e512d4ab30f5","rayStartCmd":"ray start  --address=long-running-wg4gd-head-svc.test-ns-thrrw.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:49:47.606Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"0ff0b3e8-c9b9-4196-af83-e512d4ab30f5","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-wg4gd-head-svc.test-ns-thrrw.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:49:47.606Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"0ff0b3e8-c9b9-4196-af83-e512d4ab30f5","enabled":true}
{"level":"error","ts":"2026-06-05T09:49:47.709Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"0ff0b3e8-c9b9-4196-af83-e512d4ab30f5","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-wg4gd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:47.709Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"0ff0b3e8-c9b9-4196-af83-e512d4ab30f5","error":"FailedCreateWorkerPod\npods \"long-running-wg4gd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated"}
{"level":"info","ts":"2026-06-05T09:49:47.709Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"0ff0b3e8-c9b9-4196-af83-e512d4ab30f5","podName":"long-running-wg4gd-head-jwf4t","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:49:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:49:47.709Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"0ff0b3e8-c9b9-4196-af83-e512d4ab30f5"}
{"level":"error","ts":"2026-06-05T09:49:47.709Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"0ff0b3e8-c9b9-4196-af83-e512d4ab30f5","error":"FailedCreateWorkerPod\npods \"long-running-wg4gd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:47.870Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"36266843-9eb9-46f1-9cc5-ba5d1a1d4c9a"}
{"level":"info","ts":"2026-06-05T09:49:47.870Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"36266843-9eb9-46f1-9cc5-ba5d1a1d4c9a"}
{"level":"info","ts":"2026-06-05T09:49:47.870Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"36266843-9eb9-46f1-9cc5-ba5d1a1d4c9a","1 head service found":"long-running-wg4gd-head-svc"}
{"level":"info","ts":"2026-06-05T09:49:47.870Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"36266843-9eb9-46f1-9cc5-ba5d1a1d4c9a","Found 1 head Pod":"long-running-wg4gd-head-jwf4t","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:49:47.870Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"36266843-9eb9-46f1-9cc5-ba5d1a1d4c9a","head Pod":"long-running-wg4gd-head-jwf4t","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-wg4gd-head-jwf4t. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:49:47.870Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"36266843-9eb9-46f1-9cc5-ba5d1a1d4c9a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:49:47.870Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"36266843-9eb9-46f1-9cc5-ba5d1a1d4c9a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:47.870Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"36266843-9eb9-46f1-9cc5-ba5d1a1d4c9a","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-05T09:49:47.870Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"36266843-9eb9-46f1-9cc5-ba5d1a1d4c9a","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:47.870Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"36266843-9eb9-46f1-9cc5-ba5d1a1d4c9a","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-05T09:49:47.870Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"36266843-9eb9-46f1-9cc5-ba5d1a1d4c9a"}
{"level":"info","ts":"2026-06-05T09:49:47.870Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"36266843-9eb9-46f1-9cc5-ba5d1a1d4c9a","nodeType":"worker","rayStartParams":{"address":"long-running-wg4gd-head-svc.test-ns-thrrw.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-05T09:49:47.870Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"36266843-9eb9-46f1-9cc5-ba5d1a1d4c9a","rayStartCmd":"ray start  --address=long-running-wg4gd-head-svc.test-ns-thrrw.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:49:47.870Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"36266843-9eb9-46f1-9cc5-ba5d1a1d4c9a","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-wg4gd-head-svc.test-ns-thrrw.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:49:47.870Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"36266843-9eb9-46f1-9cc5-ba5d1a1d4c9a","enabled":true}
{"level":"error","ts":"2026-06-05T09:49:47.874Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"36266843-9eb9-46f1-9cc5-ba5d1a1d4c9a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-wg4gd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:47.874Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"36266843-9eb9-46f1-9cc5-ba5d1a1d4c9a","error":"FailedCreateWorkerPod\npods \"long-running-wg4gd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated"}
{"level":"info","ts":"2026-06-05T09:49:47.874Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"36266843-9eb9-46f1-9cc5-ba5d1a1d4c9a","podName":"long-running-wg4gd-head-jwf4t","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:49:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:49:47.874Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"36266843-9eb9-46f1-9cc5-ba5d1a1d4c9a"}
{"level":"error","ts":"2026-06-05T09:49:47.874Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"36266843-9eb9-46f1-9cc5-ba5d1a1d4c9a","error":"FailedCreateWorkerPod\npods \"long-running-wg4gd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:48.195Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7c6c94ad-ff63-4e79-bc98-8542dae015d9"}
{"level":"info","ts":"2026-06-05T09:49:48.195Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7c6c94ad-ff63-4e79-bc98-8542dae015d9"}
{"level":"info","ts":"2026-06-05T09:49:48.196Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7c6c94ad-ff63-4e79-bc98-8542dae015d9","1 head service found":"long-running-wg4gd-head-svc"}
{"level":"info","ts":"2026-06-05T09:49:48.196Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7c6c94ad-ff63-4e79-bc98-8542dae015d9","Found 1 head Pod":"long-running-wg4gd-head-jwf4t","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-05T09:49:48.196Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7c6c94ad-ff63-4e79-bc98-8542dae015d9","head Pod":"long-running-wg4gd-head-jwf4t","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-wg4gd-head-jwf4t. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-05T09:49:48.196Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7c6c94ad-ff63-4e79-bc98-8542dae015d9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:49:48.196Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7c6c94ad-ff63-4e79-bc98-8542dae015d9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:48.196Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7c6c94ad-ff63-4e79-bc98-8542dae015d9","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-05T09:49:48.196Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7c6c94ad-ff63-4e79-bc98-8542dae015d9","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:48.196Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7c6c94ad-ff63-4e79-bc98-8542dae015d9","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-05T09:49:48.196Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7c6c94ad-ff63-4e79-bc98-8542dae015d9"}
{"level":"info","ts":"2026-06-05T09:49:48.196Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7c6c94ad-ff63-4e79-bc98-8542dae015d9","nodeType":"worker","rayStartParams":{"address":"long-running-wg4gd-head-svc.test-ns-thrrw.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-05T09:49:48.196Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7c6c94ad-ff63-4e79-bc98-8542dae015d9","rayStartCmd":"ray start  --address=long-running-wg4gd-head-svc.test-ns-thrrw.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:49:48.196Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7c6c94ad-ff63-4e79-bc98-8542dae015d9","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-wg4gd-head-svc.test-ns-thrrw.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:49:48.196Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7c6c94ad-ff63-4e79-bc98-8542dae015d9","enabled":true}
{"level":"error","ts":"2026-06-05T09:49:48.199Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7c6c94ad-ff63-4e79-bc98-8542dae015d9","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-wg4gd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:48.199Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7c6c94ad-ff63-4e79-bc98-8542dae015d9","error":"FailedCreateWorkerPod\npods \"long-running-wg4gd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated"}
{"level":"info","ts":"2026-06-05T09:49:48.199Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7c6c94ad-ff63-4e79-bc98-8542dae015d9","podName":"long-running-wg4gd-head-jwf4t","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-05T09:49:36Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-05T09:49:48.200Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7c6c94ad-ff63-4e79-bc98-8542dae015d9"}
{"level":"error","ts":"2026-06-05T09:49:48.200Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"7c6c94ad-ff63-4e79-bc98-8542dae015d9","error":"FailedCreateWorkerPod\npods \"long-running-wg4gd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:48.219Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"ae12bfe0-d32c-4266-8177-47d7b6124247"}
{"level":"info","ts":"2026-06-05T09:49:48.219Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"ae12bfe0-d32c-4266-8177-47d7b6124247"}
{"level":"info","ts":"2026-06-05T09:49:48.219Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"ae12bfe0-d32c-4266-8177-47d7b6124247","1 head service found":"long-running-wg4gd-head-svc"}
{"level":"info","ts":"2026-06-05T09:49:48.219Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"ae12bfe0-d32c-4266-8177-47d7b6124247","Found 1 head Pod":"long-running-wg4gd-head-jwf4t","Pod status":"Failed","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"&ContainerStateTerminated{ExitCode:1,Signal:0,Reason:Error,Message:,StartedAt:2026-06-05 09:49:42 +0000 UTC,FinishedAt:2026-06-05 09:49:48 +0000 UTC,ContainerID:cri-o://00de90d8ca6521b00392970da7b41b003ba97d879546b8c4856dc6d5b398e45b,}"}
{"level":"info","ts":"2026-06-05T09:49:48.219Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"ae12bfe0-d32c-4266-8177-47d7b6124247","head Pod":"long-running-wg4gd-head-jwf4t","shouldDelete":true,"reason":"The head Pod long-running-wg4gd-head-jwf4t status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"error","ts":"2026-06-05T09:49:48.222Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"ae12bfe0-d32c-4266-8177-47d7b6124247","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"The head Pod long-running-wg4gd-head-jwf4t status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:48.222Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"ae12bfe0-d32c-4266-8177-47d7b6124247","error":"The head Pod long-running-wg4gd-head-jwf4t status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"info","ts":"2026-06-05T09:49:48.223Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"ae12bfe0-d32c-4266-8177-47d7b6124247","podName":"long-running-wg4gd-head-jwf4t","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-06-05T09:49:48.223Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"ae12bfe0-d32c-4266-8177-47d7b6124247","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:49:33Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-05T09:49:47Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"long-running-wg4gd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:49:33Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-05T09:49:47Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"long-running-wg4gd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated"}]}
{"level":"info","ts":"2026-06-05T09:49:48.223Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"ae12bfe0-d32c-4266-8177-47d7b6124247","name":"long-running-wg4gd","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:49:47Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.26","serviceIP":"10.132.0.26","podName":"long-running-wg4gd-head-jwf4t","serviceName":"long-running-wg4gd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:49:33Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-05T09:49:47Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"long-running-wg4gd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:49:48Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.26","serviceIP":"10.132.0.26","podName":"long-running-wg4gd-head-jwf4t","serviceName":"long-running-wg4gd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:49:33Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-05T09:49:47Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"long-running-wg4gd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:49:48.234Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"ae12bfe0-d32c-4266-8177-47d7b6124247"}
{"level":"error","ts":"2026-06-05T09:49:48.234Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"ae12bfe0-d32c-4266-8177-47d7b6124247","error":"The head Pod long-running-wg4gd-head-jwf4t status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:48.234Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"203ff985-b6f6-46e0-b4e6-69a2fdb1d786","rayCluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:48.234Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"da3e47b7-fbe6-471c-aecd-02faa798088a","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:48.234Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"203ff985-b6f6-46e0-b4e6-69a2fdb1d786","pod":"long-running-wg4gd-head-jwf4t","ip":"10.132.0.26"}
{"level":"info","ts":"2026-06-05T09:49:48.234Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"203ff985-b6f6-46e0-b4e6-69a2fdb1d786"}
{"level":"info","ts":"2026-06-05T09:49:48.234Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"d52b72f3-6b7f-4f03-96ea-4738d806f554","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:48.234Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"d52b72f3-6b7f-4f03-96ea-4738d806f554","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-06-05 09:49:39 +0000 UTC","Now":"2026-06-05T09:49:48.234Z","ShutdownTime":"2026-06-05T09:59:39.000Z"}
{"level":"info","ts":"2026-06-05T09:49:48.234Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"d52b72f3-6b7f-4f03-96ea-4738d806f554","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-05T09:49:48.234Z","logger":"controllers.RayJob","msg":"shutdownTime not reached, requeue this RayJob for n seconds","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"d52b72f3-6b7f-4f03-96ea-4738d806f554","seconds":592}
{"level":"info","ts":"2026-06-05T09:49:48.239Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"da3e47b7-fbe6-471c-aecd-02faa798088a","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:48.840Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"9b107ce5-18f8-4df4-8036-b5b55c5753e8"}
{"level":"info","ts":"2026-06-05T09:49:48.840Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"9b107ce5-18f8-4df4-8036-b5b55c5753e8"}
{"level":"info","ts":"2026-06-05T09:49:48.840Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"9b107ce5-18f8-4df4-8036-b5b55c5753e8","1 head service found":"long-running-wg4gd-head-svc"}
{"level":"info","ts":"2026-06-05T09:49:48.840Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"9b107ce5-18f8-4df4-8036-b5b55c5753e8","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-06-05T09:49:48.841Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"9b107ce5-18f8-4df4-8036-b5b55c5753e8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:49:48.841Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"9b107ce5-18f8-4df4-8036-b5b55c5753e8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:48.841Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"9b107ce5-18f8-4df4-8036-b5b55c5753e8","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-05T09:49:48.841Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"9b107ce5-18f8-4df4-8036-b5b55c5753e8","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:48.841Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"9b107ce5-18f8-4df4-8036-b5b55c5753e8","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-05T09:49:48.841Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"9b107ce5-18f8-4df4-8036-b5b55c5753e8"}
{"level":"info","ts":"2026-06-05T09:49:48.841Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"9b107ce5-18f8-4df4-8036-b5b55c5753e8","nodeType":"worker","rayStartParams":{"address":"long-running-wg4gd-head-svc.test-ns-thrrw.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-05T09:49:48.841Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"9b107ce5-18f8-4df4-8036-b5b55c5753e8","rayStartCmd":"ray start  --address=long-running-wg4gd-head-svc.test-ns-thrrw.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:49:48.841Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"9b107ce5-18f8-4df4-8036-b5b55c5753e8","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-wg4gd-head-svc.test-ns-thrrw.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:49:48.841Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"9b107ce5-18f8-4df4-8036-b5b55c5753e8","enabled":true}
{"level":"error","ts":"2026-06-05T09:49:48.845Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"9b107ce5-18f8-4df4-8036-b5b55c5753e8","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-wg4gd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:48.845Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"9b107ce5-18f8-4df4-8036-b5b55c5753e8","error":"FailedCreateWorkerPod\npods \"long-running-wg4gd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated"}
{"level":"info","ts":"2026-06-05T09:49:48.845Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"9b107ce5-18f8-4df4-8036-b5b55c5753e8","podName":"long-running-wg4gd-head-jwf4t","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-06-05T09:49:48.845Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"9b107ce5-18f8-4df4-8036-b5b55c5753e8"}
{"level":"error","ts":"2026-06-05T09:49:48.845Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"9b107ce5-18f8-4df4-8036-b5b55c5753e8","error":"FailedCreateWorkerPod\npods \"long-running-wg4gd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:48.995Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"58681f98-aca0-4908-a6a0-4a7b4ae0e768"}
{"level":"info","ts":"2026-06-05T09:49:48.995Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"58681f98-aca0-4908-a6a0-4a7b4ae0e768"}
{"level":"info","ts":"2026-06-05T09:49:48.995Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"58681f98-aca0-4908-a6a0-4a7b4ae0e768","1 head service found":"long-running-wg4gd-head-svc"}
{"level":"info","ts":"2026-06-05T09:49:48.996Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"58681f98-aca0-4908-a6a0-4a7b4ae0e768","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-06-05T09:49:48.996Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"58681f98-aca0-4908-a6a0-4a7b4ae0e768","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-05T09:49:48.996Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"58681f98-aca0-4908-a6a0-4a7b4ae0e768","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:48.996Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"58681f98-aca0-4908-a6a0-4a7b4ae0e768","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-05T09:49:48.996Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"58681f98-aca0-4908-a6a0-4a7b4ae0e768","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-05T09:49:48.996Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"58681f98-aca0-4908-a6a0-4a7b4ae0e768","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-05T09:49:48.996Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"58681f98-aca0-4908-a6a0-4a7b4ae0e768"}
{"level":"info","ts":"2026-06-05T09:49:48.996Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"58681f98-aca0-4908-a6a0-4a7b4ae0e768","nodeType":"worker","rayStartParams":{"address":"long-running-wg4gd-head-svc.test-ns-thrrw.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-05T09:49:48.996Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"58681f98-aca0-4908-a6a0-4a7b4ae0e768","rayStartCmd":"ray start  --address=long-running-wg4gd-head-svc.test-ns-thrrw.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:49:48.996Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"58681f98-aca0-4908-a6a0-4a7b4ae0e768","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-wg4gd-head-svc.test-ns-thrrw.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-05T09:49:48.996Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"58681f98-aca0-4908-a6a0-4a7b4ae0e768","enabled":true}
{"level":"error","ts":"2026-06-05T09:49:48.999Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"58681f98-aca0-4908-a6a0-4a7b4ae0e768","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-wg4gd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:48.999Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"58681f98-aca0-4908-a6a0-4a7b4ae0e768","error":"FailedCreateWorkerPod\npods \"long-running-wg4gd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated"}
{"level":"info","ts":"2026-06-05T09:49:48.999Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"58681f98-aca0-4908-a6a0-4a7b4ae0e768","podName":"long-running-wg4gd-head-jwf4t","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-06-05T09:49:49.000Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"58681f98-aca0-4908-a6a0-4a7b4ae0e768"}
{"level":"error","ts":"2026-06-05T09:49:49.000Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"58681f98-aca0-4908-a6a0-4a7b4ae0e768","error":"FailedCreateWorkerPod\npods \"long-running-wg4gd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:49.001Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"d96de57c-0329-48df-9b16-55d31cfcea71"}
{"level":"info","ts":"2026-06-05T09:49:49.001Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"d96de57c-0329-48df-9b16-55d31cfcea71"}
{"level":"info","ts":"2026-06-05T09:49:49.001Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"d96de57c-0329-48df-9b16-55d31cfcea71","1 head service found":"long-running-wg4gd-head-svc"}
{"level":"info","ts":"2026-06-05T09:49:49.002Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"d96de57c-0329-48df-9b16-55d31cfcea71"}
{"level":"info","ts":"2026-06-05T09:49:49.002Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"d96de57c-0329-48df-9b16-55d31cfcea71","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:49:49.002Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"d96de57c-0329-48df-9b16-55d31cfcea71"}
{"level":"info","ts":"2026-06-05T09:49:49.002Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"d96de57c-0329-48df-9b16-55d31cfcea71","mode":"IntegratedOAuth","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:49.002Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"d96de57c-0329-48df-9b16-55d31cfcea71","cluster":"long-running-wg4gd","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-05T09:49:49.002Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"d96de57c-0329-48df-9b16-55d31cfcea71","cluster":"long-running-wg4gd","authType":"IntegratedOAuth","serviceAccount":"long-running-wg4gd-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-05T09:49:49.002Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"d96de57c-0329-48df-9b16-55d31cfcea71","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"long-running-wg4gd","ray.io/group":"headgroup","ray.io/identifier":"long-running-wg4gd-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-05T09:49:49.002Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"d96de57c-0329-48df-9b16-55d31cfcea71","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"long-running-wg4gd-head-svc.test-ns-thrrw.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-05T09:49:49.002Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"d96de57c-0329-48df-9b16-55d31cfcea71","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-wg4gd-head-svc.test-ns-thrrw.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-05T09:49:49.002Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"d96de57c-0329-48df-9b16-55d31cfcea71","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-wg4gd-head-svc.test-ns-thrrw.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-05T09:49:49.002Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"d96de57c-0329-48df-9b16-55d31cfcea71","enabled":true}
{"level":"error","ts":"2026-06-05T09:49:49.005Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"d96de57c-0329-48df-9b16-55d31cfcea71","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"long-running-wg4gd-head-\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:49.005Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"d96de57c-0329-48df-9b16-55d31cfcea71","error":"FailedCreateHeadPod\npods \"long-running-wg4gd-head-\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated"}
{"level":"info","ts":"2026-06-05T09:49:49.005Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"d96de57c-0329-48df-9b16-55d31cfcea71","filter labels":["test-ns-thrrw",{"ray.io/cluster":"long-running-wg4gd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:49:49.005Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"d96de57c-0329-48df-9b16-55d31cfcea71","filter labels":["test-ns-thrrw",{"ray.io/cluster":"long-running-wg4gd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:49:49.005Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"d96de57c-0329-48df-9b16-55d31cfcea71","filter labels":["test-ns-thrrw",{"ray.io/cluster":"long-running-wg4gd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:49:49.005Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"d96de57c-0329-48df-9b16-55d31cfcea71","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podIP":"10.132.0.26","serviceIP":"10.132.0.26","podName":"long-running-wg4gd-head-jwf4t","serviceName":"long-running-wg4gd-head-svc"},"newHead":{"serviceName":"long-running-wg4gd-head-svc"}}
{"level":"info","ts":"2026-06-05T09:49:49.005Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"d96de57c-0329-48df-9b16-55d31cfcea71","name":"long-running-wg4gd","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:49:48Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.26","serviceIP":"10.132.0.26","podName":"long-running-wg4gd-head-jwf4t","serviceName":"long-running-wg4gd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:49:33Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-05T09:49:47Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"long-running-wg4gd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:49:49Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-wg4gd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:49:33Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:49:33Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-05T09:49:47Z","reason":"FailedCreateHeadPod","message":"FailedCreateHeadPod\npods \"long-running-wg4gd-head-\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-05T09:49:49.019Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"d96de57c-0329-48df-9b16-55d31cfcea71"}
{"level":"info","ts":"2026-06-05T09:49:49.019Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"3dbb1d24-18bb-4874-bf7f-535e1e0f6db7","cluster":"long-running-wg4gd"}
{"level":"error","ts":"2026-06-05T09:49:49.019Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"d96de57c-0329-48df-9b16-55d31cfcea71","error":"FailedCreateHeadPod\npods \"long-running-wg4gd-head-\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:49.019Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"d549e319-7776-44aa-8d92-d2fb4534571b","rayCluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:49.019Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"b5667642-f3d4-442e-b4f8-01714bd516e9","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:49.019Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"b5667642-f3d4-442e-b4f8-01714bd516e9","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-06-05 09:49:39 +0000 UTC","Now":"2026-06-05T09:49:49.019Z","ShutdownTime":"2026-06-05T09:59:39.000Z"}
{"level":"info","ts":"2026-06-05T09:49:49.019Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"b5667642-f3d4-442e-b4f8-01714bd516e9","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-05T09:49:49.019Z","logger":"controllers.RayJob","msg":"shutdownTime not reached, requeue this RayJob for n seconds","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"b5667642-f3d4-442e-b4f8-01714bd516e9","seconds":591}
{"level":"info","ts":"2026-06-05T09:49:49.025Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"3dbb1d24-18bb-4874-bf7f-535e1e0f6db7","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:49.036Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"d549e319-7776-44aa-8d92-d2fb4534571b","certificate":"ray-head-cert-long-running-wg4gd","dnsNames":["long-running-wg4gd-head-svc","localhost","long-running-wg4gd-head-svc.test-ns-thrrw.svc","long-running-wg4gd-head-svc.test-ns-thrrw.svc.cluster.local"],"ipAddresses":["127.0.0.1"]}
{"level":"info","ts":"2026-06-05T09:49:49.052Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"d549e319-7776-44aa-8d92-d2fb4534571b","certificate":"ray-worker-cert-long-running-wg4gd","dnsNames":["long-running-wg4gd-worker-svc","localhost","long-running-wg4gd-small-group","long-running-wg4gd-worker-svc.test-ns-thrrw.svc","long-running-wg4gd-worker-svc.test-ns-thrrw.svc.cluster.local","*.long-running-wg4gd-worker-svc.test-ns-thrrw.svc","*.long-running-wg4gd-worker-svc.test-ns-thrrw.svc.cluster.local","*-worker-*.test-ns-thrrw.svc","*-worker-*.test-ns-thrrw.svc.cluster.local"],"ipAddresses":["127.0.0.1"]}
{"level":"info","ts":"2026-06-05T09:49:49.052Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"d549e319-7776-44aa-8d92-d2fb4534571b"}
{"level":"error","ts":"2026-06-05T09:49:49.267Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-thrrw"},"reconcileID":"d0ed6296-cc48-45e9-abbf-39bceab32a72","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:49.267Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-thrrw"},"reconcileID":"d0ed6296-cc48-45e9-abbf-39bceab32a72"}
{"level":"error","ts":"2026-06-05T09:49:49.267Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-thrrw"},"reconcileID":"d0ed6296-cc48-45e9-abbf-39bceab32a72","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:51.126Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"8114c174-e5c8-43c3-b18f-c4bdb6ef00a6","namespacedName":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:49:51.126Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"8114c174-e5c8-43c3-b18f-c4bdb6ef00a6","mode":"IntegratedOAuth","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:51.126Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"8114c174-e5c8-43c3-b18f-c4bdb6ef00a6","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:51.134Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"dc211311-ba78-4617-b103-2d7b98218977"}
{"level":"info","ts":"2026-06-05T09:49:51.134Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"dc211311-ba78-4617-b103-2d7b98218977"}
{"level":"error","ts":"2026-06-05T09:49:51.137Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"dc211311-ba78-4617-b103-2d7b98218977","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcileHeadService-fm","error":"services \"long-running-wg4gd-head-svc\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:51.138Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"dc211311-ba78-4617-b103-2d7b98218977","error":"services \"long-running-wg4gd-head-svc\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated"}
{"level":"info","ts":"2026-06-05T09:49:51.138Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"dc211311-ba78-4617-b103-2d7b98218977","filter labels":["test-ns-thrrw",{"ray.io/cluster":"long-running-wg4gd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:49:51.138Z","logger":"controllers.RayCluster","msg":"updateEndpoints: Unable to find a Service for this RayCluster. Not adding RayCluster status.endpoints","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"dc211311-ba78-4617-b103-2d7b98218977","serviceSelectors":["test-ns-thrrw",{"ray.io/cluster":"long-running-wg4gd","ray.io/identifier":"long-running-wg4gd-head","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:49:51.138Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"dc211311-ba78-4617-b103-2d7b98218977","filter labels":["test-ns-thrrw",{"ray.io/cluster":"long-running-wg4gd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:49:51.138Z","logger":"controllers.RayCluster","msg":"Got error when calculating new status","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"dc211311-ba78-4617-b103-2d7b98218977","error":"unable to find head service. cluster name long-running-wg4gd, filter labels [test-ns-thrrw map[ray.io/cluster:long-running-wg4gd ray.io/identifier:long-running-wg4gd-head ray.io/node-type:head]]"}
{"level":"info","ts":"2026-06-05T09:49:51.138Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"dc211311-ba78-4617-b103-2d7b98218977"}
{"level":"error","ts":"2026-06-05T09:49:51.138Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"dc211311-ba78-4617-b103-2d7b98218977","error":"services \"long-running-wg4gd-head-svc\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-05T09:49:51.143Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"8114c174-e5c8-43c3-b18f-c4bdb6ef00a6","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"long-running-wg4gd-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:51.143Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"8114c174-e5c8-43c3-b18f-c4bdb6ef00a6"}
{"level":"error","ts":"2026-06-05T09:49:51.143Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"8114c174-e5c8-43c3-b18f-c4bdb6ef00a6","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"long-running-wg4gd-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:51.143Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"85aca4ca-968e-437e-854b-a4cd14eb8fcc","namespacedName":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:49:51.143Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"85aca4ca-968e-437e-854b-a4cd14eb8fcc","mode":"IntegratedOAuth","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:51.143Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"85aca4ca-968e-437e-854b-a4cd14eb8fcc","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:51.143Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"7fdb2362-73ce-45d4-9334-8ccabc12ea7c","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:51.144Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"1fd551a2-018e-4a2d-baa1-54534a50288a","rayCluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:51.144Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"1fd551a2-018e-4a2d-baa1-54534a50288a"}
{"level":"info","ts":"2026-06-05T09:49:51.144Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"5110403a-e051-4641-9cbc-4ca7c53091e3","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:51.144Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"5110403a-e051-4641-9cbc-4ca7c53091e3","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-06-05 09:49:39 +0000 UTC","Now":"2026-06-05T09:49:51.144Z","ShutdownTime":"2026-06-05T09:59:39.000Z"}
{"level":"info","ts":"2026-06-05T09:49:51.144Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"5110403a-e051-4641-9cbc-4ca7c53091e3","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-05T09:49:51.144Z","logger":"controllers.RayJob","msg":"shutdownTime not reached, requeue this RayJob for n seconds","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"5110403a-e051-4641-9cbc-4ca7c53091e3","seconds":589}
{"level":"info","ts":"2026-06-05T09:49:51.148Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"7fdb2362-73ce-45d4-9334-8ccabc12ea7c","cluster":"long-running-wg4gd"}
{"level":"error","ts":"2026-06-05T09:49:51.151Z","logger":"authentication-controller","msg":"Failed to update AuthenticationReady condition","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"85aca4ca-968e-437e-854b-a4cd14eb8fcc","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-wg4gd\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleOIDCConfiguration\n\t/workspace/controllers/ray/authentication_controller.go:299\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:174\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-05T09:49:51.151Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"85aca4ca-968e-437e-854b-a4cd14eb8fcc","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"long-running-wg4gd-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:51.151Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"85aca4ca-968e-437e-854b-a4cd14eb8fcc"}
{"level":"error","ts":"2026-06-05T09:49:51.151Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"85aca4ca-968e-437e-854b-a4cd14eb8fcc","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"long-running-wg4gd-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:51.151Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"b781a783-f16c-43d8-9e26-10dd1b22dd71","namespacedName":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:49:51.151Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"b781a783-f16c-43d8-9e26-10dd1b22dd71","mode":"IntegratedOAuth","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:51.151Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"b781a783-f16c-43d8-9e26-10dd1b22dd71","cluster":"long-running-wg4gd"}
{"level":"error","ts":"2026-06-05T09:49:51.164Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"b781a783-f16c-43d8-9e26-10dd1b22dd71","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"long-running-wg4gd-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:51.164Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"b781a783-f16c-43d8-9e26-10dd1b22dd71"}
{"level":"error","ts":"2026-06-05T09:49:51.164Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"b781a783-f16c-43d8-9e26-10dd1b22dd71","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"long-running-wg4gd-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:51.184Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"ae830f3b-f407-49ae-9972-3e736b2e453a","namespacedName":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:49:51.184Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"ae830f3b-f407-49ae-9972-3e736b2e453a","mode":"IntegratedOAuth","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:51.184Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"ae830f3b-f407-49ae-9972-3e736b2e453a","cluster":"long-running-wg4gd"}
{"level":"error","ts":"2026-06-05T09:49:51.196Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"ae830f3b-f407-49ae-9972-3e736b2e453a","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"long-running-wg4gd-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:51.196Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"ae830f3b-f407-49ae-9972-3e736b2e453a"}
{"level":"error","ts":"2026-06-05T09:49:51.196Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"ae830f3b-f407-49ae-9972-3e736b2e453a","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"long-running-wg4gd-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:51.237Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"6b8df2a3-0fac-43ba-b038-f3797c0e0225","namespacedName":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:49:51.237Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"6b8df2a3-0fac-43ba-b038-f3797c0e0225","mode":"IntegratedOAuth","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:51.237Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"6b8df2a3-0fac-43ba-b038-f3797c0e0225","cluster":"long-running-wg4gd"}
{"level":"error","ts":"2026-06-05T09:49:51.248Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"6b8df2a3-0fac-43ba-b038-f3797c0e0225","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"long-running-wg4gd-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:51.248Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"6b8df2a3-0fac-43ba-b038-f3797c0e0225"}
{"level":"error","ts":"2026-06-05T09:49:51.248Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"6b8df2a3-0fac-43ba-b038-f3797c0e0225","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"long-running-wg4gd-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:51.330Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"299aff74-ff3b-4521-898b-b3783cc550df","namespacedName":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:49:51.330Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"299aff74-ff3b-4521-898b-b3783cc550df","mode":"IntegratedOAuth","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:51.330Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"299aff74-ff3b-4521-898b-b3783cc550df","cluster":"long-running-wg4gd"}
{"level":"error","ts":"2026-06-05T09:49:51.343Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"299aff74-ff3b-4521-898b-b3783cc550df","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"long-running-wg4gd-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:51.343Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"299aff74-ff3b-4521-898b-b3783cc550df"}
{"level":"error","ts":"2026-06-05T09:49:51.343Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"299aff74-ff3b-4521-898b-b3783cc550df","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"long-running-wg4gd-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:51.406Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"ca89f7b2-2db6-4153-9636-6bfbaa7f6f3a"}
{"level":"info","ts":"2026-06-05T09:49:51.406Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"ca89f7b2-2db6-4153-9636-6bfbaa7f6f3a"}
{"level":"error","ts":"2026-06-05T09:49:51.408Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"ca89f7b2-2db6-4153-9636-6bfbaa7f6f3a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcileHeadService-fm","error":"services \"long-running-wg4gd-head-svc\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:51.409Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"ca89f7b2-2db6-4153-9636-6bfbaa7f6f3a","error":"services \"long-running-wg4gd-head-svc\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated"}
{"level":"info","ts":"2026-06-05T09:49:51.409Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"ca89f7b2-2db6-4153-9636-6bfbaa7f6f3a","filter labels":["test-ns-thrrw",{"ray.io/cluster":"long-running-wg4gd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:49:51.409Z","logger":"controllers.RayCluster","msg":"updateEndpoints: Unable to find a Service for this RayCluster. Not adding RayCluster status.endpoints","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"ca89f7b2-2db6-4153-9636-6bfbaa7f6f3a","serviceSelectors":["test-ns-thrrw",{"ray.io/cluster":"long-running-wg4gd","ray.io/identifier":"long-running-wg4gd-head","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:49:51.409Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"ca89f7b2-2db6-4153-9636-6bfbaa7f6f3a","filter labels":["test-ns-thrrw",{"ray.io/cluster":"long-running-wg4gd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-05T09:49:51.409Z","logger":"controllers.RayCluster","msg":"Got error when calculating new status","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"ca89f7b2-2db6-4153-9636-6bfbaa7f6f3a","error":"unable to find head service. cluster name long-running-wg4gd, filter labels [test-ns-thrrw map[ray.io/cluster:long-running-wg4gd ray.io/identifier:long-running-wg4gd-head ray.io/node-type:head]]"}
{"level":"info","ts":"2026-06-05T09:49:51.409Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"ca89f7b2-2db6-4153-9636-6bfbaa7f6f3a"}
{"level":"error","ts":"2026-06-05T09:49:51.409Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"ca89f7b2-2db6-4153-9636-6bfbaa7f6f3a","error":"services \"long-running-wg4gd-head-svc\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:51.415Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"d9bf4100-5149-4c1e-a021-37e5f048da09","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:51.421Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"d9bf4100-5149-4c1e-a021-37e5f048da09","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:51.421Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"f50d9d0d-4282-4548-9e53-5419fa285226","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:51.426Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"f50d9d0d-4282-4548-9e53-5419fa285226","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:51.504Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"8b3ca3fe-7bcf-4b0f-96af-be82beb09ca8","namespacedName":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:49:51.504Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"8b3ca3fe-7bcf-4b0f-96af-be82beb09ca8","mode":"IntegratedOAuth","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:51.504Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"8b3ca3fe-7bcf-4b0f-96af-be82beb09ca8","cluster":"long-running-wg4gd"}
{"level":"error","ts":"2026-06-05T09:49:51.517Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"8b3ca3fe-7bcf-4b0f-96af-be82beb09ca8","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"long-running-wg4gd-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:51.517Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"8b3ca3fe-7bcf-4b0f-96af-be82beb09ca8"}
{"level":"error","ts":"2026-06-05T09:49:51.517Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"8b3ca3fe-7bcf-4b0f-96af-be82beb09ca8","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"long-running-wg4gd-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:51.790Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"95cb24ff-ff5c-4ecf-818a-36fbac26631a","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:51.790Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"95cb24ff-ff5c-4ecf-818a-36fbac26631a","name":"raycluster"}
{"level":"info","ts":"2026-06-05T09:49:51.790Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"95cb24ff-ff5c-4ecf-818a-36fbac26631a","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"}}
{"level":"error","ts":"2026-06-05T09:49:51.799Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"95cb24ff-ff5c-4ecf-818a-36fbac26631a","JobId":"counter-65xg8","error":"Job counter-65xg8 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:51.799Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"95cb24ff-ff5c-4ecf-818a-36fbac26631a"}
{"level":"error","ts":"2026-06-05T09:49:51.799Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"95cb24ff-ff5c-4ecf-818a-36fbac26631a","error":"Job counter-65xg8 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:51.838Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"6c12737a-6328-4b50-98b6-58376a47994f","namespacedName":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:49:51.838Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"6c12737a-6328-4b50-98b6-58376a47994f","mode":"IntegratedOAuth","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:51.838Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"6c12737a-6328-4b50-98b6-58376a47994f","cluster":"long-running-wg4gd"}
{"level":"error","ts":"2026-06-05T09:49:51.851Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"6c12737a-6328-4b50-98b6-58376a47994f","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"long-running-wg4gd-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:51.851Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"6c12737a-6328-4b50-98b6-58376a47994f"}
{"level":"error","ts":"2026-06-05T09:49:51.851Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"6c12737a-6328-4b50-98b6-58376a47994f","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"long-running-wg4gd-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:52.492Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"06c9546e-b523-4d0e-974d-ac0c4f817c3b","namespacedName":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:49:52.492Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"06c9546e-b523-4d0e-974d-ac0c4f817c3b","mode":"IntegratedOAuth","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:52.492Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"06c9546e-b523-4d0e-974d-ac0c4f817c3b","cluster":"long-running-wg4gd"}
{"level":"error","ts":"2026-06-05T09:49:52.504Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"06c9546e-b523-4d0e-974d-ac0c4f817c3b","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"long-running-wg4gd-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:52.504Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"06c9546e-b523-4d0e-974d-ac0c4f817c3b"}
{"level":"error","ts":"2026-06-05T09:49:52.504Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"06c9546e-b523-4d0e-974d-ac0c4f817c3b","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"long-running-wg4gd-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:53.227Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"49cbc72a-d165-4b2a-a0ed-c20203c24060","namespacedName":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:49:53.227Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"8bbcc87a-faf5-4721-abaa-d7ce271646ab","rayCluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:53.227Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"reconcileID":"5debd542-95f6-4da1-860c-e097a85fabc5"}
{"level":"info","ts":"2026-06-05T09:49:53.227Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"49cbc72a-d165-4b2a-a0ed-c20203c24060","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:53.227Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"49cbc72a-d165-4b2a-a0ed-c20203c24060","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:53.227Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"8bbcc87a-faf5-4721-abaa-d7ce271646ab","rayCluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:53.227Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"1d7812b6-8d08-48c5-90a7-c893bc9f54bc"}
{"level":"info","ts":"2026-06-05T09:49:53.227Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"49cbc72a-d165-4b2a-a0ed-c20203c24060","namespace":"test-ns-thrrw","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-05T09:49:53.227Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"49cbc72a-d165-4b2a-a0ed-c20203c24060","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-06-05T09:49:53.227Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"d47c008f-df9d-4853-8dd0-de5d3062afa0","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:53.227Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"d47c008f-df9d-4853-8dd0-de5d3062afa0","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-06-05 09:49:39 +0000 UTC","Now":"2026-06-05T09:49:53.227Z","ShutdownTime":"2026-06-05T09:59:39.000Z"}
{"level":"info","ts":"2026-06-05T09:49:53.227Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"d47c008f-df9d-4853-8dd0-de5d3062afa0","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-05T09:49:53.227Z","logger":"controllers.RayJob","msg":"shutdownTime not reached, requeue this RayJob for n seconds","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"d47c008f-df9d-4853-8dd0-de5d3062afa0","seconds":587}
{"level":"error","ts":"2026-06-05T09:49:53.230Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"8bbcc87a-faf5-4721-abaa-d7ce271646ab","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-wg4gd\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-05T09:49:53.230Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"8bbcc87a-faf5-4721-abaa-d7ce271646ab","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-wg4gd\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:53.230Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"8bbcc87a-faf5-4721-abaa-d7ce271646ab"}
{"level":"error","ts":"2026-06-05T09:49:53.230Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"8bbcc87a-faf5-4721-abaa-d7ce271646ab","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-wg4gd\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:53.233Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"49cbc72a-d165-4b2a-a0ed-c20203c24060","name":"test-ns-thrrw-long-running-wg4gd","namespace":"opendatahub"}
{"level":"info","ts":"2026-06-05T09:49:53.236Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"353d4a00-11f5-497e-bf6f-f3e073c37bc3","rayCluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:53.236Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"353d4a00-11f5-497e-bf6f-f3e073c37bc3","rayCluster":"long-running-wg4gd"}
{"level":"error","ts":"2026-06-05T09:49:53.239Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"353d4a00-11f5-497e-bf6f-f3e073c37bc3","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-wg4gd\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-05T09:49:53.239Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"353d4a00-11f5-497e-bf6f-f3e073c37bc3","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-wg4gd\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:53.239Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"353d4a00-11f5-497e-bf6f-f3e073c37bc3"}
{"level":"error","ts":"2026-06-05T09:49:53.239Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"353d4a00-11f5-497e-bf6f-f3e073c37bc3","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-wg4gd\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:53.239Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"49cbc72a-d165-4b2a-a0ed-c20203c24060","configMap":"kube-rbac-proxy-config-long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:53.247Z","logger":"raycluster-resource","msg":"default","name":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:53.247Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-wg4gd","namespace":"test-ns-thrrw"}
{"level":"info","ts":"2026-06-05T09:49:53.250Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"0299cde1-dda4-4e41-87ab-7c1fa9123a34","rayCluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:53.250Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"0299cde1-dda4-4e41-87ab-7c1fa9123a34","rayCluster":"long-running-wg4gd"}
{"level":"error","ts":"2026-06-05T09:49:53.253Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"0299cde1-dda4-4e41-87ab-7c1fa9123a34","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-wg4gd\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-05T09:49:53.253Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"0299cde1-dda4-4e41-87ab-7c1fa9123a34","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-wg4gd\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:53.253Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"0299cde1-dda4-4e41-87ab-7c1fa9123a34"}
{"level":"error","ts":"2026-06-05T09:49:53.253Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"0299cde1-dda4-4e41-87ab-7c1fa9123a34","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-wg4gd\" is forbidden: unable to create new content in namespace test-ns-thrrw because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:53.258Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"a0d8b7f9-ac28-44d0-8471-019ecbdee601"}
{"level":"info","ts":"2026-06-05T09:49:53.258Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"c98ed112-1518-4625-b40d-89ddcc5cb5c3","namespace":"test-ns-thrrw","clusterName":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:53.258Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"5171013c-c40a-4a33-bf66-9980bdfc9856","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:49:53.258Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"5171013c-c40a-4a33-bf66-9980bdfc9856","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-06-05 09:49:39 +0000 UTC","Now":"2026-06-05T09:49:53.258Z","ShutdownTime":"2026-06-05T09:59:39.000Z"}
{"level":"info","ts":"2026-06-05T09:49:53.258Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"5171013c-c40a-4a33-bf66-9980bdfc9856","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-05T09:49:53.258Z","logger":"controllers.RayJob","msg":"shutdownTime not reached, requeue this RayJob for n seconds","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"5171013c-c40a-4a33-bf66-9980bdfc9856","seconds":587}
{"level":"info","ts":"2026-06-05T09:49:53.259Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"49cbc72a-d165-4b2a-a0ed-c20203c24060","cluster":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:53.259Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"58d4333a-9773-4241-9f22-04b72f62f584","namespacedName":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:49:53.259Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"58d4333a-9773-4241-9f22-04b72f62f584"}
{"level":"info","ts":"2026-06-05T09:49:53.259Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"58d4333a-9773-4241-9f22-04b72f62f584","namespace":"test-ns-thrrw","authClusters":0}
{"level":"info","ts":"2026-06-05T09:49:53.259Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"58d4333a-9773-4241-9f22-04b72f62f584","namespace":"test-ns-thrrw"}
{"level":"error","ts":"2026-06-05T09:49:53.268Z","msg":"Failed to delete worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"c98ed112-1518-4625-b40d-89ddcc5cb5c3","certificate":"ray-worker-cert-long-running-wg4gd","error":"certificates.cert-manager.io \"ray-worker-cert-long-running-wg4gd\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:268\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-05T09:49:53.306Z","msg":"Failed to delete CA certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"c98ed112-1518-4625-b40d-89ddcc5cb5c3","certificate":"ray-ca-certificate-long-running-wg4gd","error":"certificates.cert-manager.io \"ray-ca-certificate-long-running-wg4gd\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:275\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:53.306Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"c98ed112-1518-4625-b40d-89ddcc5cb5c3","errorCount":2}
{"level":"info","ts":"2026-06-05T09:49:53.306Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"c98ed112-1518-4625-b40d-89ddcc5cb5c3"}
{"level":"error","ts":"2026-06-05T09:49:53.306Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"c98ed112-1518-4625-b40d-89ddcc5cb5c3","error":"certificates.cert-manager.io \"ray-worker-cert-long-running-wg4gd\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:53.306Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"7575061e-1f74-4e18-a8fc-fe97fd0d3b1c","namespace":"test-ns-thrrw","clusterName":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:53.306Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"7575061e-1f74-4e18-a8fc-fe97fd0d3b1c","namespace":"test-ns-thrrw","clusterName":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:53.346Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"e78f138b-30d1-468a-88bb-d2ede2add23f","namespace":"test-ns-thrrw","clusterName":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:53.346Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"e78f138b-30d1-468a-88bb-d2ede2add23f","namespace":"test-ns-thrrw","clusterName":"long-running-wg4gd"}
{"level":"info","ts":"2026-06-05T09:49:53.618Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-thrrw"},"reconcileID":"f526cef7-0706-41ae-8c48-5472370e190e"}
{"level":"info","ts":"2026-06-05T09:49:53.628Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"54897800-2428-40b0-90ad-910980bad2f8","DeletionTimestamp":"2026-06-05 09:49:53 +0000 UTC"}
{"level":"error","ts":"2026-06-05T09:49:53.628Z","logger":"controllers.RayJob","msg":"Failed to get RayCluster","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"54897800-2428-40b0-90ad-910980bad2f8","error":"RayCluster.ray.io \"long-running-wg4gd\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:115\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:53.628Z","logger":"controllers.RayJob","msg":"Stop a ray job","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"54897800-2428-40b0-90ad-910980bad2f8","rayJob":"long-running-77jkz"}
{"level":"error","ts":"2026-06-05T09:49:53.628Z","logger":"controllers.RayJob","msg":"Failed to stop job for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"54897800-2428-40b0-90ad-910980bad2f8","error":"Post \"http:///api/jobs/long-running-77jkz/stop\": http: no Host in request URL","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:123\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:49:53.628Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"54897800-2428-40b0-90ad-910980bad2f8","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-05T09:49:53.643Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"b24e68f2-cc95-457c-ab77-c2a5b581ba8e"}
{"level":"info","ts":"2026-06-05T09:49:53.784Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"0dc2ca3f-75fc-4ea8-be4b-a6a8adc29302","namespacedName":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"}}
{"level":"info","ts":"2026-06-05T09:49:53.785Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"0dc2ca3f-75fc-4ea8-be4b-a6a8adc29302"}
{"level":"info","ts":"2026-06-05T09:49:53.785Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"0dc2ca3f-75fc-4ea8-be4b-a6a8adc29302","namespace":"test-ns-thrrw","authClusters":0}
{"level":"info","ts":"2026-06-05T09:49:53.785Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-wg4gd","namespace":"test-ns-thrrw"},"namespace":"test-ns-thrrw","name":"long-running-wg4gd","reconcileID":"0dc2ca3f-75fc-4ea8-be4b-a6a8adc29302","namespace":"test-ns-thrrw"}
{"level":"info","ts":"2026-06-05T09:49:56.644Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"long-running","namespace":"test-ns-thrrw"},"reconcileID":"ad8c9d15-357e-43c1-90da-b15ce6238960"}
{"level":"info","ts":"2026-06-05T09:50:02.039Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"16fa03b7-f782-4159-86c1-3ae3c0d5ae8a","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:50:02.039Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"16fa03b7-f782-4159-86c1-3ae3c0d5ae8a","name":"raycluster"}
{"level":"info","ts":"2026-06-05T09:50:02.039Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"16fa03b7-f782-4159-86c1-3ae3c0d5ae8a","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"}}
{"level":"info","ts":"2026-06-05T09:50:02.047Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"16fa03b7-f782-4159-86c1-3ae3c0d5ae8a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-65xg8","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-rfwvh.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-05T09:49:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:49:52Z","endTime":"2026-06-05T09:49:55Z"},"jobId":"counter-65xg8","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-rfwvh.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-05T09:49:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:01Z","stateTransitionTimes":{"ready":"2026-06-05T09:39:01Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.20","serviceIP":"10.134.0.20","podName":"raycluster-head-vjd7d","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:38:59Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:39:01Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:50:02.047Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"16fa03b7-f782-4159-86c1-3ae3c0d5ae8a","old JobStatus":"","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-05T09:50:02.060Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"d243f1e7-24e1-4ece-b616-f9fdc02a6a2a","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:50:02.060Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"d243f1e7-24e1-4ece-b616-f9fdc02a6a2a","name":"raycluster"}
{"level":"info","ts":"2026-06-05T09:50:02.060Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"d243f1e7-24e1-4ece-b616-f9fdc02a6a2a","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"}}
{"level":"info","ts":"2026-06-05T09:50:02.063Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"d243f1e7-24e1-4ece-b616-f9fdc02a6a2a","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:49:52Z","endTime":"2026-06-05T09:49:55Z"},"jobId":"counter-65xg8","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-rfwvh.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-05T09:49:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:01Z","stateTransitionTimes":{"ready":"2026-06-05T09:39:01Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.20","serviceIP":"10.134.0.20","podName":"raycluster-head-vjd7d","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:38:59Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:39:01Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:49:52Z","endTime":"2026-06-05T09:49:55Z"},"jobId":"counter-65xg8","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-rfwvh.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-05T09:49:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:01Z","stateTransitionTimes":{"ready":"2026-06-05T09:39:01Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.20","serviceIP":"10.134.0.20","podName":"raycluster-head-vjd7d","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:38:59Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:39:01Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:50:03.036Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"6e063b9d-3ae9-43a0-b789-f14813b49fea","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:50:03.036Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"6e063b9d-3ae9-43a0-b789-f14813b49fea","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-05T09:50:03.042Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"6e063b9d-3ae9-43a0-b789-f14813b49fea"}
{"level":"info","ts":"2026-06-05T09:50:03.042Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"6e063b9d-3ae9-43a0-b789-f14813b49fea","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-05T09:50:03.042Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"6e063b9d-3ae9-43a0-b789-f14813b49fea","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-vfl6r","rayClusterName":"raycluster","jobDeploymentStatus":"Initializing","startTime":"2026-06-05T09:50:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:50:03.042Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"6e063b9d-3ae9-43a0-b789-f14813b49fea","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-05T09:50:03.052Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"03d6ef0d-f92c-4e54-a7e2-16fb12770802","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:50:03.052Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"03d6ef0d-f92c-4e54-a7e2-16fb12770802","name":"raycluster"}
{"level":"info","ts":"2026-06-05T09:50:03.052Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"03d6ef0d-f92c-4e54-a7e2-16fb12770802","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"}}
{"level":"info","ts":"2026-06-05T09:50:03.052Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"03d6ef0d-f92c-4e54-a7e2-16fb12770802","head service name":"raycluster-head-svc","namespace":"test-ns-rfwvh"}
{"level":"info","ts":"2026-06-05T09:50:03.052Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"03d6ef0d-f92c-4e54-a7e2-16fb12770802","head service URL":"raycluster-head-svc.test-ns-rfwvh.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-05T09:50:03.052Z","logger":"controllers.RayJob","msg":"user-provided submitter template is used; the first container is assumed to be the submitter","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"03d6ef0d-f92c-4e54-a7e2-16fb12770802"}
{"level":"info","ts":"2026-06-05T09:50:03.052Z","logger":"controllers.RayJob","msg":"No command is specified in the user-provided template. Default command is used","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"03d6ef0d-f92c-4e54-a7e2-16fb12770802","command":["if","!","ray","job","status","--address","http://raycluster-head-svc.test-ns-rfwvh.svc.cluster.local:8265","fail-vfl6r",">/dev/null","2>&1",";","then","ray","job","submit","--address","http://raycluster-head-svc.test-ns-rfwvh.svc.cluster.local:8265","--no-wait","--submission-id","fail-vfl6r","--","python /home/ray/jobs/fail.py",";","fi",";","ray","job","logs","--address","http://raycluster-head-svc.test-ns-rfwvh.svc.cluster.local:8265","--follow","fail-vfl6r"]}
{"level":"info","ts":"2026-06-05T09:50:03.061Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"03d6ef0d-f92c-4e54-a7e2-16fb12770802","Kubernetes Job":"fail"}
{"level":"info","ts":"2026-06-05T09:50:03.061Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"03d6ef0d-f92c-4e54-a7e2-16fb12770802","SubmissionMode":"K8sJobMode","RayCluster":"raycluster"}
{"level":"info","ts":"2026-06-05T09:50:03.061Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"03d6ef0d-f92c-4e54-a7e2-16fb12770802","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-vfl6r","rayClusterName":"raycluster","jobDeploymentStatus":"Initializing","startTime":"2026-06-05T09:50:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-vfl6r","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-rfwvh.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-05T09:50:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-05T09:50:03.061Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"03d6ef0d-f92c-4e54-a7e2-16fb12770802","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-05T09:50:03.069Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"ca54a584-1d4a-47f2-967b-366d750c493a","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:50:03.069Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"ca54a584-1d4a-47f2-967b-366d750c493a","name":"raycluster"}
{"level":"info","ts":"2026-06-05T09:50:03.069Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"ca54a584-1d4a-47f2-967b-366d750c493a","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"}}
{"level":"error","ts":"2026-06-05T09:50:03.072Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"ca54a584-1d4a-47f2-967b-366d750c493a","JobId":"fail-vfl6r","error":"Job fail-vfl6r does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:50:03.072Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"ca54a584-1d4a-47f2-967b-366d750c493a"}
{"level":"error","ts":"2026-06-05T09:50:03.072Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"ca54a584-1d4a-47f2-967b-366d750c493a","error":"Job fail-vfl6r does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:50:03.072Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"6df9040a-2390-4c43-bf75-6c1e81b5364e","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:50:03.072Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"6df9040a-2390-4c43-bf75-6c1e81b5364e","name":"raycluster"}
{"level":"info","ts":"2026-06-05T09:50:03.072Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"6df9040a-2390-4c43-bf75-6c1e81b5364e","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"}}
{"level":"error","ts":"2026-06-05T09:50:03.077Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"6df9040a-2390-4c43-bf75-6c1e81b5364e","JobId":"fail-vfl6r","error":"Job fail-vfl6r does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:50:03.077Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"6df9040a-2390-4c43-bf75-6c1e81b5364e"}
{"level":"error","ts":"2026-06-05T09:50:03.077Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"6df9040a-2390-4c43-bf75-6c1e81b5364e","error":"Job fail-vfl6r does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:50:03.077Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"a4962ef1-5e8b-46d9-ae20-a2ad8b20696e","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:50:03.077Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"a4962ef1-5e8b-46d9-ae20-a2ad8b20696e","name":"raycluster"}
{"level":"info","ts":"2026-06-05T09:50:03.077Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"a4962ef1-5e8b-46d9-ae20-a2ad8b20696e","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"}}
{"level":"error","ts":"2026-06-05T09:50:03.082Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"a4962ef1-5e8b-46d9-ae20-a2ad8b20696e","JobId":"fail-vfl6r","error":"Job fail-vfl6r does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:50:03.082Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"a4962ef1-5e8b-46d9-ae20-a2ad8b20696e"}
{"level":"error","ts":"2026-06-05T09:50:03.082Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"a4962ef1-5e8b-46d9-ae20-a2ad8b20696e","error":"Job fail-vfl6r does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:50:03.082Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"100e3919-0167-4baa-9c68-accb5ff9ace9","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:50:03.082Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"100e3919-0167-4baa-9c68-accb5ff9ace9","name":"raycluster"}
{"level":"info","ts":"2026-06-05T09:50:03.082Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"100e3919-0167-4baa-9c68-accb5ff9ace9","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"}}
{"level":"error","ts":"2026-06-05T09:50:03.105Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"100e3919-0167-4baa-9c68-accb5ff9ace9","JobId":"fail-vfl6r","error":"Job fail-vfl6r does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:50:03.106Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"100e3919-0167-4baa-9c68-accb5ff9ace9"}
{"level":"error","ts":"2026-06-05T09:50:03.106Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"100e3919-0167-4baa-9c68-accb5ff9ace9","error":"Job fail-vfl6r does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:50:03.106Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"db262d28-78bc-4e1f-bc68-9c9f058ebfe5","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:50:03.106Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"db262d28-78bc-4e1f-bc68-9c9f058ebfe5","name":"raycluster"}
{"level":"info","ts":"2026-06-05T09:50:03.106Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"db262d28-78bc-4e1f-bc68-9c9f058ebfe5","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"}}
{"level":"error","ts":"2026-06-05T09:50:03.111Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"db262d28-78bc-4e1f-bc68-9c9f058ebfe5","JobId":"fail-vfl6r","error":"Job fail-vfl6r does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:50:03.111Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"db262d28-78bc-4e1f-bc68-9c9f058ebfe5"}
{"level":"error","ts":"2026-06-05T09:50:03.111Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"db262d28-78bc-4e1f-bc68-9c9f058ebfe5","error":"Job fail-vfl6r does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:50:03.146Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"f84226ed-0400-4bb8-9879-6ca5fe8117ab","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:50:03.146Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"f84226ed-0400-4bb8-9879-6ca5fe8117ab","name":"raycluster"}
{"level":"info","ts":"2026-06-05T09:50:03.146Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"f84226ed-0400-4bb8-9879-6ca5fe8117ab","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"}}
{"level":"error","ts":"2026-06-05T09:50:03.151Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"f84226ed-0400-4bb8-9879-6ca5fe8117ab","JobId":"fail-vfl6r","error":"Job fail-vfl6r does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:50:03.151Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"f84226ed-0400-4bb8-9879-6ca5fe8117ab"}
{"level":"error","ts":"2026-06-05T09:50:03.151Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"f84226ed-0400-4bb8-9879-6ca5fe8117ab","error":"Job fail-vfl6r does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:50:03.312Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"3d4a6e23-72d3-49ec-8c88-ff637406d954","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:50:03.312Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"3d4a6e23-72d3-49ec-8c88-ff637406d954","name":"raycluster"}
{"level":"info","ts":"2026-06-05T09:50:03.312Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"3d4a6e23-72d3-49ec-8c88-ff637406d954","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"}}
{"level":"error","ts":"2026-06-05T09:50:03.317Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"3d4a6e23-72d3-49ec-8c88-ff637406d954","JobId":"fail-vfl6r","error":"Job fail-vfl6r does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:50:03.317Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"3d4a6e23-72d3-49ec-8c88-ff637406d954"}
{"level":"error","ts":"2026-06-05T09:50:03.317Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"3d4a6e23-72d3-49ec-8c88-ff637406d954","error":"Job fail-vfl6r does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:50:03.384Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"6abd69eb-48ad-4f4e-a4db-1acd15d9713d","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:50:03.384Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"6abd69eb-48ad-4f4e-a4db-1acd15d9713d","name":"raycluster"}
{"level":"info","ts":"2026-06-05T09:50:03.384Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"6abd69eb-48ad-4f4e-a4db-1acd15d9713d","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"}}
{"level":"info","ts":"2026-06-05T09:50:03.390Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"6abd69eb-48ad-4f4e-a4db-1acd15d9713d","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:49:52Z","endTime":"2026-06-05T09:49:55Z"},"jobId":"counter-65xg8","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-rfwvh.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-05T09:49:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:01Z","stateTransitionTimes":{"ready":"2026-06-05T09:39:01Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.20","serviceIP":"10.134.0.20","podName":"raycluster-head-vjd7d","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:38:59Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:39:01Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:49:52Z","endTime":"2026-06-05T09:49:55Z"},"jobId":"counter-65xg8","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-rfwvh.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-05T09:49:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:01Z","stateTransitionTimes":{"ready":"2026-06-05T09:39:01Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.20","serviceIP":"10.134.0.20","podName":"raycluster-head-vjd7d","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:38:59Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:39:01Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:50:03.638Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"cddf9ab9-c6e7-4e26-aed9-39eec9277696","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:50:03.638Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"cddf9ab9-c6e7-4e26-aed9-39eec9277696","name":"raycluster"}
{"level":"info","ts":"2026-06-05T09:50:03.638Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"cddf9ab9-c6e7-4e26-aed9-39eec9277696","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"}}
{"level":"error","ts":"2026-06-05T09:50:03.641Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"cddf9ab9-c6e7-4e26-aed9-39eec9277696","JobId":"fail-vfl6r","error":"Job fail-vfl6r does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:50:03.641Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"cddf9ab9-c6e7-4e26-aed9-39eec9277696"}
{"level":"error","ts":"2026-06-05T09:50:03.641Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"cddf9ab9-c6e7-4e26-aed9-39eec9277696","error":"Job fail-vfl6r does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:50:04.081Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"ab4a98d4-1055-4d07-bd70-f3155d721926","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:50:04.082Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"ab4a98d4-1055-4d07-bd70-f3155d721926","name":"raycluster"}
{"level":"info","ts":"2026-06-05T09:50:04.082Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"ab4a98d4-1055-4d07-bd70-f3155d721926","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"}}
{"level":"error","ts":"2026-06-05T09:50:04.087Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"ab4a98d4-1055-4d07-bd70-f3155d721926","JobId":"fail-vfl6r","error":"Job fail-vfl6r does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:50:04.087Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"ab4a98d4-1055-4d07-bd70-f3155d721926"}
{"level":"error","ts":"2026-06-05T09:50:04.087Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"ab4a98d4-1055-4d07-bd70-f3155d721926","error":"Job fail-vfl6r does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:50:04.282Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"a373745c-8fa7-44d9-9edb-aa104b4085ac","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:50:04.282Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"a373745c-8fa7-44d9-9edb-aa104b4085ac","name":"raycluster"}
{"level":"info","ts":"2026-06-05T09:50:04.282Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"a373745c-8fa7-44d9-9edb-aa104b4085ac","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"}}
{"level":"error","ts":"2026-06-05T09:50:04.287Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"a373745c-8fa7-44d9-9edb-aa104b4085ac","JobId":"fail-vfl6r","error":"Job fail-vfl6r does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:50:04.287Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"a373745c-8fa7-44d9-9edb-aa104b4085ac"}
{"level":"error","ts":"2026-06-05T09:50:04.287Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"a373745c-8fa7-44d9-9edb-aa104b4085ac","error":"Job fail-vfl6r does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:50:04.393Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"a992ed77-c9af-44dd-b388-ef8f98dc44a3","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:50:04.393Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"a992ed77-c9af-44dd-b388-ef8f98dc44a3","name":"raycluster"}
{"level":"info","ts":"2026-06-05T09:50:04.393Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"a992ed77-c9af-44dd-b388-ef8f98dc44a3","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"}}
{"level":"info","ts":"2026-06-05T09:50:04.398Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"a992ed77-c9af-44dd-b388-ef8f98dc44a3","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:49:52Z","endTime":"2026-06-05T09:49:55Z"},"jobId":"counter-65xg8","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-rfwvh.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-05T09:49:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:01Z","stateTransitionTimes":{"ready":"2026-06-05T09:39:01Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.20","serviceIP":"10.134.0.20","podName":"raycluster-head-vjd7d","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:38:59Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:39:01Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:49:52Z","endTime":"2026-06-05T09:49:55Z"},"jobId":"counter-65xg8","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-rfwvh.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-05T09:49:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:01Z","stateTransitionTimes":{"ready":"2026-06-05T09:39:01Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.20","serviceIP":"10.134.0.20","podName":"raycluster-head-vjd7d","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:38:59Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:39:01Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:50:04.411Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"f413396a-f69d-4e73-9f16-e34453cbfe61","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:50:04.411Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"f413396a-f69d-4e73-9f16-e34453cbfe61","name":"raycluster"}
{"level":"info","ts":"2026-06-05T09:50:04.411Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"f413396a-f69d-4e73-9f16-e34453cbfe61","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"}}
{"level":"info","ts":"2026-06-05T09:50:04.414Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"f413396a-f69d-4e73-9f16-e34453cbfe61","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:49:52Z","endTime":"2026-06-05T09:49:55Z"},"jobId":"counter-65xg8","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-rfwvh.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-05T09:49:43Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:01Z","stateTransitionTimes":{"ready":"2026-06-05T09:39:01Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.20","serviceIP":"10.134.0.20","podName":"raycluster-head-vjd7d","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:38:59Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:39:01Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:49:52Z","endTime":"2026-06-05T09:49:55Z"},"jobId":"counter-65xg8","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-rfwvh.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Complete","message":"Job finished successfully.","startTime":"2026-06-05T09:49:43Z","succeeded":1,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:01Z","stateTransitionTimes":{"ready":"2026-06-05T09:39:01Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.20","serviceIP":"10.134.0.20","podName":"raycluster-head-vjd7d","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:38:59Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:39:01Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:50:04.414Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"f413396a-f69d-4e73-9f16-e34453cbfe61","old JobStatus":"SUCCEEDED","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Complete"}
{"level":"info","ts":"2026-06-05T09:50:04.427Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"5cb7fd6f-aaab-4ea7-89c8-8a27b144af86","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:50:04.427Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"5cb7fd6f-aaab-4ea7-89c8-8a27b144af86","ShutdownAfterJobFinishes":false,"ClusterSelector":{"ray.io/cluster":"raycluster"},"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-05 09:50:04 +0000 UTC","Now":"2026-06-05T09:50:04.427Z","ShutdownTime":"2026-06-05T09:50:04.000Z"}
{"level":"info","ts":"2026-06-05T09:50:05.060Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"a73926e2-c59c-4b88-8057-87ae5ca5775b","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:50:05.060Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-rfwvh"},"reconcileID":"a73926e2-c59c-4b88-8057-87ae5ca5775b","ShutdownAfterJobFinishes":false,"ClusterSelector":{"ray.io/cluster":"raycluster"},"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-05 09:50:04 +0000 UTC","Now":"2026-06-05T09:50:05.060Z","ShutdownTime":"2026-06-05T09:50:04.000Z"}
{"level":"info","ts":"2026-06-05T09:50:06.847Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"3df26d88-c80a-4a1d-b9a6-e7c45234e84f","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:50:06.847Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"3df26d88-c80a-4a1d-b9a6-e7c45234e84f","name":"raycluster"}
{"level":"info","ts":"2026-06-05T09:50:06.847Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"3df26d88-c80a-4a1d-b9a6-e7c45234e84f","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"}}
{"level":"error","ts":"2026-06-05T09:50:06.850Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"3df26d88-c80a-4a1d-b9a6-e7c45234e84f","JobId":"fail-vfl6r","error":"Job fail-vfl6r does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:50:06.850Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"3df26d88-c80a-4a1d-b9a6-e7c45234e84f"}
{"level":"error","ts":"2026-06-05T09:50:06.850Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"3df26d88-c80a-4a1d-b9a6-e7c45234e84f","error":"Job fail-vfl6r does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-05T09:50:09.747Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-thrrw"},"reconcileID":"83a92489-114a-492d-af91-e6cdfe64c264"}
{"level":"info","ts":"2026-06-05T09:50:11.971Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"f973c2c6-a1d2-41ec-b3ce-235d6dbf131d","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:50:11.971Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"f973c2c6-a1d2-41ec-b3ce-235d6dbf131d","name":"raycluster"}
{"level":"info","ts":"2026-06-05T09:50:11.971Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"f973c2c6-a1d2-41ec-b3ce-235d6dbf131d","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"}}
{"level":"info","ts":"2026-06-05T09:50:11.977Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"f973c2c6-a1d2-41ec-b3ce-235d6dbf131d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-vfl6r","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-rfwvh.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-05T09:50:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:50:09Z","endTime":"2026-06-05T09:50:11Z"},"jobId":"fail-vfl6r","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-rfwvh.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-05 09:50:09,781\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-vfl6r: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-05T09:50:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:01Z","stateTransitionTimes":{"ready":"2026-06-05T09:39:01Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.20","serviceIP":"10.134.0.20","podName":"raycluster-head-vjd7d","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:38:59Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:39:01Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-05T09:50:11.977Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"f973c2c6-a1d2-41ec-b3ce-235d6dbf131d","old JobStatus":"","new JobStatus":"FAILED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-05T09:50:11.988Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"2e5f4a20-47d4-41a5-b509-52f0626ba60c","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-05T09:50:11.988Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"2e5f4a20-47d4-41a5-b509-52f0626ba60c","name":"raycluster"}
{"level":"info","ts":"2026-06-05T09:50:11.988Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"2e5f4a20-47d4-41a5-b509-52f0626ba60c","RayCluster":{"name":"raycluster","namespace":"test-ns-rfwvh"}}
{"level":"info","ts":"2026-06-05T09:50:11.991Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-rfwvh"},"reconcileID":"2e5f4a20-47d4-41a5-b509-52f0626ba60c","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:50:09Z","endTime":"2026-06-05T09:50:11Z"},"jobId":"fail-vfl6r","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-rfwvh.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-05 09:50:09,781\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-vfl6r: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-05T09:50:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:01Z","stateTransitionTimes":{"ready":"2026-06-05T09:39:01Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.20","serviceIP":"10.134.0.20","podName":"raycluster-head-vjd7d","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:38:59Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:39:01Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-05T09:50:09Z","endTime":"2026-06-05T09:50:11Z"},"jobId":"fail-vfl6r","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-rfwvh.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-05 09:50:09,781\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-vfl6r: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-05T09:50:03Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-05T09:39:01Z","stateTransitionTimes":{"ready":"2026-06-05T09:39:01Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.20","serviceIP":"10.134.0.20","podName":"raycluster-head-vjd7d","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-05T09:38:59Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-05T09:39:01Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-05T09:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-05T09:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}