{"level":"Level(-2)","ts":"2026-11-04T04:05:04.497773308Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-min-interval","reconcileID":"e87a0a41-80ff-4480-8efe-05c11ed78564","job":"rhai-e2e-progression-hsgdd/progression-min-interval","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:04.589817283Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-custom-config","reconcileID":"4a3af1f1-891b-4083-8530-b5802c185d20","job":"rhai-e2e-progression-hsgdd/progression-custom-config","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:04.740605134Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-max-interval","reconcileID":"d5ab577a-5c7e-4abc-9acd-e14600f68c49","job":"rhai-e2e-progression-hsgdd/progression-max-interval","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:04.917806178Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-custom-config","reconcileID":"bd14a032-b9f4-4cba-af73-7ef5ab29a123","job":"rhai-e2e-progression-hsgdd/progression-custom-config","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:04.984061211Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-default-interval","reconcileID":"ee90b85f-2768-4d7d-860d-d0f9715b8e78","job":"rhai-e2e-progression-hsgdd/progression-default-interval","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:05.316449115Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-job-failure","reconcileID":"47bbbd40-0118-4e3b-ad4d-f2b3ce17db9e","job":"rhai-e2e-progression-hsgdd/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:06.263300298Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-custom-config","reconcileID":"d8d4c23f-fc12-4089-9adb-2b553d80098d","job":"rhai-e2e-progression-hsgdd/progression-custom-config","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:07.605308197Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-job-failure","reconcileID":"f3a3e780-c2a5-419a-8033-664a8642531f","job":"rhai-e2e-progression-hsgdd/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:07.838479124Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-job-failure","reconcileID":"5b106cae-4333-4b8c-abf6-e30129d8fbe9","job":"rhai-e2e-progression-hsgdd/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:07.958223704Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-job-failure","reconcileID":"3da821ab-8820-421b-b033-f2ba0d5a9e09","job":"rhai-e2e-progression-hsgdd/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:08.949975691Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-custom-config","reconcileID":"0beed383-bf9e-4523-bbe5-fe1b3e78f347","job":"rhai-e2e-progression-hsgdd/progression-custom-config","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:09.517682333Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-invalid","reconcileID":"161e5cce-c2fb-4982-98ae-92743c7753fb","job":"rhai-e2e-progression-hsgdd/progression-invalid","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:10.027420631Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-job-failure","reconcileID":"4b5f20ec-964a-4717-8e1e-f87d18959aa0","job":"rhai-e2e-progression-hsgdd/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:10.307928372Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-job-failure","reconcileID":"52e2e0cd-5e3e-419e-b17d-ca062046d993","job":"rhai-e2e-progression-hsgdd/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:10.526870149Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-invalid","reconcileID":"270de6ff-b600-46a0-8a46-27a5275aba39","job":"rhai-e2e-progression-hsgdd/progression-invalid","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:10.630700479Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-job-failure","reconcileID":"e7025189-d6a7-4b7e-b6bb-c7d8736f35f1","job":"rhai-e2e-progression-hsgdd/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:10.692970063Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-invalid","reconcileID":"54d8d4da-f16c-459f-b25a-b4c1318861e2","job":"rhai-e2e-progression-hsgdd/progression-invalid","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:11.219225166Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-custom-config","reconcileID":"b54b77a6-cd28-4dff-9cad-79559c9b2e39","job":"rhai-e2e-progression-hsgdd/progression-custom-config","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:11.818930315Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-invalid","reconcileID":"c389b397-2f12-4ddc-ba26-fcb74286b9c8","job":"rhai-e2e-progression-hsgdd/progression-invalid","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:12.463461336Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-job-failure","reconcileID":"e45e50ed-5ecc-4843-9c1d-60d46b271693","job":"rhai-e2e-progression-hsgdd/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:12.736460928Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-job-failure","reconcileID":"e04c525a-4c18-415f-a1ef-fda9313424bd","job":"rhai-e2e-progression-hsgdd/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:13.54634033Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-custom-config","reconcileID":"8baca4d5-c903-4022-a9ad-095d8f5b7e33","job":"rhai-e2e-progression-hsgdd/progression-custom-config","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:14.898056212Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-job-failure","reconcileID":"2d0a65a7-5b4c-400e-9a4c-62fac12e8f58","job":"rhai-e2e-progression-hsgdd/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:15.71003585Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-custom-config","reconcileID":"2aaa942a-c2d1-4dd8-96e0-c839b6414c47","job":"rhai-e2e-progression-hsgdd/progression-custom-config","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:17.001505844Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-job-failure","reconcileID":"1b41a63e-a577-4a32-aba3-ad40a5d16f35","job":"rhai-e2e-progression-hsgdd/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:17.866496575Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-custom-config","reconcileID":"cb0d79b9-9ab4-4553-8349-923fdc0e9f74","job":"rhai-e2e-progression-hsgdd/progression-custom-config","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:19.57297322Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-job-failure","reconcileID":"81b5bee0-7ac6-479b-b746-48625785ee88","job":"rhai-e2e-progression-hsgdd/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:20.22675826Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-custom-config","reconcileID":"dd71a342-74a6-4519-a562-c42da72d9869","job":"rhai-e2e-progression-hsgdd/progression-custom-config","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:20.582429678Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-job-failure","reconcileID":"b74c6e54-f4d0-4ea2-888a-74129daf6a53","job":"rhai-e2e-progression-hsgdd/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:20.686045096Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-job-failure","reconcileID":"f0e00c08-38b2-4451-b98f-ab37f82cda75","job":"rhai-e2e-progression-hsgdd/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:21.401962103Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-job-failure","reconcileID":"b325cb7c-8fae-4b0d-b248-5bcc76e8e8bb","job":"rhai-e2e-progression-hsgdd/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:21.676109034Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-job-failure","reconcileID":"d2df985a-bb76-4c33-94f4-5cf8b3dac58a","job":"rhai-e2e-progression-hsgdd/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:21.84221195Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-no-metrics","reconcileID":"d560eb34-f6b5-491c-b494-7141efb34741","job":"rhai-e2e-progression-hsgdd/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:22.533647875Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-custom-config","reconcileID":"481613a2-d539-4024-b5d2-1bddadd5c364","job":"rhai-e2e-progression-hsgdd/progression-custom-config","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:22.860486226Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-job-failure","reconcileID":"40a5db36-e5dc-45e9-9544-47fa9db8f577","job":"rhai-e2e-progression-hsgdd/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:24.208002957Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-no-metrics","reconcileID":"231a08d4-20e3-46fe-92c4-e71b0b3e8092","job":"rhai-e2e-progression-hsgdd/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:24.441283203Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-no-metrics","reconcileID":"fbce1c83-dd67-475d-a7f5-bb7dd6f83ca3","job":"rhai-e2e-progression-hsgdd/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:24.554476464Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-no-metrics","reconcileID":"c4898cc2-bc54-403a-a85c-d7082cacdea2","job":"rhai-e2e-progression-hsgdd/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:24.616960304Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-custom-config","reconcileID":"828ef142-f406-4fd2-bc85-e04cc8217129","job":"rhai-e2e-progression-hsgdd/progression-custom-config","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:24.83670218Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-custom-config","reconcileID":"d51bcbcc-d95d-4a00-8b2c-858d60df751f","job":"rhai-e2e-progression-hsgdd/progression-custom-config","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:25.606651584Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-custom-config","reconcileID":"239dc526-39b0-4b94-bf2a-2527ff4ee862","job":"rhai-e2e-progression-hsgdd/progression-custom-config","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:25.727472035Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-custom-config","reconcileID":"ea27a8a3-ff74-443c-a769-856ecdcb7dfb","job":"rhai-e2e-progression-hsgdd/progression-custom-config","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:26.262444795Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-job-failure","reconcileID":"228389b1-5d31-471f-8e3f-bed5064e2ba9","job":"rhai-e2e-progression-hsgdd/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:26.377372644Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-no-metrics","reconcileID":"c374a453-15a4-4d5e-a10f-274177efa318","job":"rhai-e2e-progression-hsgdd/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:26.611308183Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-no-metrics","reconcileID":"28266c97-e88b-4b35-9dd7-89a0b1d0f898","job":"rhai-e2e-progression-hsgdd/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:27.571976313Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-custom-config","reconcileID":"bd86c8db-e506-4efa-9f5a-43dcc9d9dda1","job":"rhai-e2e-progression-hsgdd/progression-custom-config","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:27.956064893Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-job-failure","reconcileID":"6f03d2e0-f43f-416e-be33-400620110aa6","job":"rhai-e2e-progression-hsgdd/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:28.071702921Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-no-metrics","reconcileID":"721e1f21-0fdc-41c3-9fde-c05d540ca262","job":"rhai-e2e-progression-hsgdd/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:29.107970928Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-job-failure","reconcileID":"4d81faa9-fcea-4488-ad2c-9b8485ca11ee","job":"rhai-e2e-progression-hsgdd/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:30.135516905Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-job-failure","reconcileID":"dfbf8fbd-c25f-4248-84b6-5f76cbf16bfe","job":"rhai-e2e-progression-hsgdd/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:30.946290536Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-job-failure","reconcileID":"bf295eaa-7b21-438c-80a4-1bf7767142c6","job":"rhai-e2e-progression-hsgdd/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:31.276140478Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-job-failure","reconcileID":"b8ee827c-7645-457c-aedd-5e555d9bd5a7","job":"rhai-e2e-progression-hsgdd/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:31.638903862Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-no-metrics","reconcileID":"c1209060-a921-4af7-a79a-70b94e64dddc","job":"rhai-e2e-progression-hsgdd/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:31.920438342Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-no-metrics","reconcileID":"e86e3feb-e18b-4408-9aea-4242e684bcd5","job":"rhai-e2e-progression-hsgdd/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:32.649129283Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-no-metrics","reconcileID":"f7bf3d51-470e-4caa-b7e3-110a7a0a7cc9","job":"rhai-e2e-progression-hsgdd/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:32.67903784Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-job-failure","reconcileID":"c71b0583-44df-484f-9121-21a17fff08b0","job":"rhai-e2e-progression-hsgdd/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:32.714984772Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-no-metrics","reconcileID":"10a4019a-b97e-4a28-9e0d-af616d8c072a","job":"rhai-e2e-progression-hsgdd/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:32.793132493Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-no-metrics","reconcileID":"959df1af-471c-4c0c-a111-758e1f20dbc9","job":"rhai-e2e-progression-hsgdd/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:33.881308929Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-job-failure","reconcileID":"e7f4cf93-4ff9-4be5-a8ff-7f99a1189a26","job":"rhai-e2e-progression-hsgdd/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:34.533782804Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-job-failure","reconcileID":"1d5b303b-0bb2-4d8c-87b2-e13545239231","job":"rhai-e2e-progression-hsgdd/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:39.820336143Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-custom-config","reconcileID":"154f9848-c0b4-476b-b976-5ec37ac4567f","job":"rhai-e2e-progression-hsgdd/progression-custom-config","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:39.826467247Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-disabled","reconcileID":"70178f05-b06a-47eb-ab0a-e577466ea338","job":"rhai-e2e-progression-hsgdd/progression-disabled","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:39.832041107Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-enabled","reconcileID":"7e70d376-15fb-47c9-875e-c9eb4125bc8c","job":"rhai-e2e-progression-hsgdd/progression-enabled","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:39.837660184Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-invalid","reconcileID":"a8caf753-a311-4a1e-8f82-d186cf69fc1d","job":"rhai-e2e-progression-hsgdd/progression-invalid","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:39.842528849Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-job-failure","reconcileID":"80405e05-450f-4cfd-9243-a251f269f5b3","job":"rhai-e2e-progression-hsgdd/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:05:39.848353022Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-hsgdd","name":"progression-no-metrics","reconcileID":"46b0118d-1428-45d9-88e3-551ffc9f6db7","job":"rhai-e2e-progression-hsgdd/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-hsgdd"}
{"level":"Level(-2)","ts":"2026-11-04T04:06:42.73252378Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-9j6g4","name":"test-dra-trainjob-n55pp","reconcileID":"04ef1979-4fed-47b5-a9ca-7b6dc78178fc","job":"test-ns-9j6g4/test-dra-trainjob-n55pp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-9j6g4"}
{"level":"Level(-2)","ts":"2026-11-04T04:06:42.980953626Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-9j6g4","name":"test-dra-trainjob-n55pp","reconcileID":"a0a08d5a-2277-4563-aeb6-373955d8bf94","job":"test-ns-9j6g4/test-dra-trainjob-n55pp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-9j6g4"}
{"level":"Level(-2)","ts":"2026-11-04T04:06:43.211334136Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-9j6g4","name":"test-dra-trainjob-n55pp","reconcileID":"82d8c229-0423-43fb-826b-62f335327a08","job":"test-ns-9j6g4/test-dra-trainjob-n55pp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-9j6g4"}
{"level":"Level(-2)","ts":"2026-11-04T04:06:43.325543155Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-9j6g4","name":"test-dra-trainjob-n55pp","reconcileID":"4d7e9061-e4f2-4e98-a861-f7b350c21c08","job":"test-ns-9j6g4/test-dra-trainjob-n55pp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-9j6g4"}
{"level":"Level(-2)","ts":"2026-11-04T04:06:43.462567985Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-9j6g4","name":"test-dra-trainjob-n55pp","reconcileID":"0cd3cba4-3c80-4521-8beb-67854a63ae88","job":"test-ns-9j6g4/test-dra-trainjob-n55pp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-9j6g4"}
{"level":"Level(-2)","ts":"2026-11-04T04:06:43.576573095Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-9j6g4","name":"test-dra-trainjob-n55pp","reconcileID":"6c401fe2-b736-46a2-b594-fa2eafef0542","job":"test-ns-9j6g4/test-dra-trainjob-n55pp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-9j6g4"}
{"level":"Level(-2)","ts":"2026-11-04T04:06:47.325283596Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-5d9h2","name":"test-trainjob-dkbxt","reconcileID":"43feb235-aa28-448c-85dd-b7b0226e6a56","job":"test-ns-5d9h2/test-trainjob-dkbxt","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-5d9h2"}
{"level":"Level(-2)","ts":"2026-11-04T04:06:47.562015085Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-5d9h2","name":"test-trainjob-dkbxt","reconcileID":"daa36502-9bbe-4fe8-86bc-8a97e2c24581","job":"test-ns-5d9h2/test-trainjob-dkbxt","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-5d9h2"}
{"level":"Level(-2)","ts":"2026-11-04T04:06:47.799513037Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-5d9h2","name":"test-trainjob-dkbxt","reconcileID":"c2a647c4-4473-4ed7-9531-eff56861b53d","job":"test-ns-5d9h2/test-trainjob-dkbxt","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-5d9h2"}
{"level":"Level(-2)","ts":"2026-11-04T04:06:47.912700603Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-5d9h2","name":"test-trainjob-dkbxt","reconcileID":"5341b545-15f3-4218-b827-209e45d9d6b9","job":"test-ns-5d9h2/test-trainjob-dkbxt","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-5d9h2"}
{"level":"Level(-2)","ts":"2026-11-04T04:06:47.912775887Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-5d9h2","name":"test-trainjob-dkbxt","reconcileID":"3d77200e-125e-4d68-87aa-ec9ea5977f97","job":"test-ns-5d9h2/test-trainjob-dkbxt","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-5d9h2"}
{"level":"Level(-2)","ts":"2026-11-04T04:06:48.027728781Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-5d9h2","name":"test-trainjob-dkbxt","reconcileID":"2ff1aa81-2e6c-43bf-9edc-2537e8c5903b","job":"test-ns-5d9h2/test-trainjob-dkbxt","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-5d9h2"}
{"level":"Level(-2)","ts":"2026-11-04T04:06:50.002689393Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-9j6g4","name":"test-dra-trainjob-n55pp","reconcileID":"0ae7dc11-417b-4bbf-9753-a0795081e08b","job":"test-ns-9j6g4/test-dra-trainjob-n55pp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-9j6g4"}
{"level":"Level(-2)","ts":"2026-11-04T04:06:50.105301641Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-5d9h2","name":"test-trainjob-dkbxt","reconcileID":"5e2de1ef-7a74-4cf4-9ac9-cb241e35f22d","job":"test-ns-5d9h2/test-trainjob-dkbxt","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-5d9h2"}
{"level":"Level(-2)","ts":"2026-11-04T04:06:50.119227572Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-9j6g4","name":"test-dra-trainjob-n55pp","reconcileID":"5d656d92-6023-485e-8306-786048798c50","job":"test-ns-9j6g4/test-dra-trainjob-n55pp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-9j6g4"}
{"level":"Level(-2)","ts":"2026-11-04T04:06:50.234021674Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-5d9h2","name":"test-trainjob-dkbxt","reconcileID":"bfc3eddd-21cf-44aa-b291-dc7184ac2180","job":"test-ns-5d9h2/test-trainjob-dkbxt","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-5d9h2"}
{"level":"Level(-2)","ts":"2026-11-04T04:06:54.123571582Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-9j6g4","name":"test-dra-trainjob-n55pp","reconcileID":"76612f7c-2602-4611-af32-bfa1e67685af","job":"test-ns-9j6g4/test-dra-trainjob-n55pp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-9j6g4"}
{"level":"Level(-2)","ts":"2026-11-04T04:06:58.056871814Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-5d9h2","name":"test-trainjob-dkbxt","reconcileID":"b27920e5-9222-45ea-8ad4-daab3ff4bce6","job":"test-ns-5d9h2/test-trainjob-dkbxt","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-5d9h2"}
{"level":"Level(-2)","ts":"2026-11-04T04:06:58.169293488Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-5d9h2","name":"test-trainjob-dkbxt","reconcileID":"4404c4b4-0888-4a4a-8448-5253dbf23eb2","job":"test-ns-5d9h2/test-trainjob-dkbxt","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-5d9h2"}
{"level":"Level(-2)","ts":"2026-11-04T04:06:59.066692562Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-5d9h2","name":"test-trainjob-dkbxt","reconcileID":"9485fc93-b966-48be-9283-eb6161ce0f9d","job":"test-ns-5d9h2/test-trainjob-dkbxt","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-5d9h2"}
{"level":"Level(-2)","ts":"2026-11-04T04:06:59.132675918Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-5d9h2","name":"test-trainjob-dkbxt","reconcileID":"4329954e-aa32-430b-b5e3-d3662c2e7b21","job":"test-ns-5d9h2/test-trainjob-dkbxt","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-5d9h2"}
{"level":"Level(-2)","ts":"2026-11-04T04:06:59.181465441Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-5d9h2","name":"test-trainjob-dkbxt","reconcileID":"c2d66a94-4a94-4a0c-a1ad-4a01a730f232","job":"test-ns-5d9h2/test-trainjob-dkbxt","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-5d9h2"}
{"level":"Level(-2)","ts":"2026-11-04T04:06:59.294989167Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-5d9h2","name":"test-trainjob-dkbxt","reconcileID":"fd13cf33-e012-4c63-8fdb-833406761d1a","job":"test-ns-5d9h2/test-trainjob-dkbxt","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-5d9h2"}
{"level":"Level(-2)","ts":"2026-11-04T04:06:59.698061841Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-dfftl","name":"test-trainjob-r6kqs","reconcileID":"0fd29d0c-e040-4816-8f40-2a9ca1926b52","job":"test-ns-dfftl/test-trainjob-r6kqs","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-dfftl"}
{"level":"Level(-2)","ts":"2026-11-04T04:06:59.935971486Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-dfftl","name":"test-trainjob-r6kqs","reconcileID":"a8acc9ff-8f25-4034-abde-7074097f8674","job":"test-ns-dfftl/test-trainjob-r6kqs","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-dfftl"}
{"level":"Level(-2)","ts":"2026-11-04T04:07:00.165602246Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-dfftl","name":"test-trainjob-r6kqs","reconcileID":"24e319a5-6a70-474a-81d6-0bbacd74e7ad","job":"test-ns-dfftl/test-trainjob-r6kqs","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-dfftl"}
{"level":"Level(-2)","ts":"2026-11-04T04:07:00.278720908Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-dfftl","name":"test-trainjob-r6kqs","reconcileID":"88fd7849-7336-44d6-907c-db2d03116239","job":"test-ns-dfftl/test-trainjob-r6kqs","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-dfftl"}
{"level":"Level(-2)","ts":"2026-11-04T04:07:00.278805268Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-dfftl","name":"test-trainjob-r6kqs","reconcileID":"dbf5cf47-a9e8-4aa7-8aea-249b66ae6ada","job":"test-ns-dfftl/test-trainjob-r6kqs","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-dfftl"}
{"level":"Level(-2)","ts":"2026-11-04T04:07:00.392724629Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-dfftl","name":"test-trainjob-r6kqs","reconcileID":"bb277c24-d327-415c-bd59-8d35855460f6","job":"test-ns-dfftl/test-trainjob-r6kqs","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-dfftl"}
{"level":"Level(-2)","ts":"2026-11-04T04:12:32.615243084Z","logger":"tas-node-reconciler","caller":"tas/node_controller.go:136","msg":"Reconcile Node","replica-role":"leader","name":"ip-10-0-135-81.us-west-2.compute.internal","reconcileID":"544ad152-80d2-4d4b-a8ed-5f54a5da5024"}
{"level":"Level(-2)","ts":"2026-11-04T04:13:08.077918259Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-dfftl","name":"test-trainjob-r6kqs","reconcileID":"6029cb71-872d-4800-9f8e-240d172262cc","job":"test-ns-dfftl/test-trainjob-r6kqs","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-dfftl"}
{"level":"Level(-2)","ts":"2026-11-04T04:13:08.207414862Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-dfftl","name":"test-trainjob-r6kqs","reconcileID":"7f733426-dfc4-49fd-a059-713b120b99ec","job":"test-ns-dfftl/test-trainjob-r6kqs","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-dfftl"}
{"level":"Level(-2)","ts":"2026-11-04T04:13:13.037454634Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-dfftl","name":"test-trainjob-r6kqs","reconcileID":"3059d0d1-f9c7-40e7-9342-047360cad838","job":"test-ns-dfftl/test-trainjob-r6kqs","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-dfftl"}
{"level":"Level(-2)","ts":"2026-11-04T04:13:13.151588147Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-dfftl","name":"test-trainjob-r6kqs","reconcileID":"72737f96-4f5f-4b4c-9e3c-db2c7702eadd","job":"test-ns-dfftl/test-trainjob-r6kqs","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-dfftl"}
{"level":"Level(-2)","ts":"2026-11-04T04:13:14.04617356Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-dfftl","name":"test-trainjob-r6kqs","reconcileID":"2ecede12-3776-448c-80f2-f09a1109cad9","job":"test-ns-dfftl/test-trainjob-r6kqs","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-dfftl"}
{"level":"Level(-2)","ts":"2026-11-04T04:13:14.113085359Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-dfftl","name":"test-trainjob-r6kqs","reconcileID":"8f52f22e-f7ee-4eb4-a693-ee60e2288af6","job":"test-ns-dfftl/test-trainjob-r6kqs","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-dfftl"}
{"level":"Level(-2)","ts":"2026-11-04T04:13:14.160228694Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-dfftl","name":"test-trainjob-r6kqs","reconcileID":"313e9dae-9f58-4680-aebb-4c06c80e1f91","job":"test-ns-dfftl/test-trainjob-r6kqs","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-dfftl"}
{"level":"Level(-2)","ts":"2026-11-04T04:13:14.274979337Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-dfftl","name":"test-trainjob-r6kqs","reconcileID":"d72d2e40-46a5-4e29-84e0-71f65312047b","job":"test-ns-dfftl/test-trainjob-r6kqs","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-dfftl"}
{"level":"Level(-2)","ts":"2026-11-04T04:13:15.56667829Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ljr2h","name":"test-trainjob-tt5jj","reconcileID":"429ef675-5929-40a3-8b1f-a47d12f7f522","job":"test-ns-ljr2h/test-trainjob-tt5jj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ljr2h"}
{"level":"Level(-2)","ts":"2026-11-04T04:13:16.046649478Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ljr2h","name":"test-trainjob-tt5jj","reconcileID":"0b7b3edc-b7d3-45e9-8e7c-8485a3ebe276","job":"test-ns-ljr2h/test-trainjob-tt5jj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ljr2h"}
{"level":"Level(-2)","ts":"2026-11-04T04:13:16.276665077Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ljr2h","name":"test-trainjob-tt5jj","reconcileID":"1a76de53-cd70-4e8e-8a5d-1cf36a11cbc2","job":"test-ns-ljr2h/test-trainjob-tt5jj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ljr2h"}
{"level":"Level(-2)","ts":"2026-11-04T04:13:16.390648631Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ljr2h","name":"test-trainjob-tt5jj","reconcileID":"d93b2864-214e-4dc8-97d0-52d6e4ac7b93","job":"test-ns-ljr2h/test-trainjob-tt5jj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ljr2h"}
{"level":"Level(-2)","ts":"2026-11-04T04:13:16.579482197Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ljr2h","name":"test-trainjob-tt5jj","reconcileID":"cd604ada-7a58-4abc-8787-4ffb07cd55e3","job":"test-ns-ljr2h/test-trainjob-tt5jj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ljr2h"}
{"level":"Level(-2)","ts":"2026-11-04T04:13:16.691925846Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ljr2h","name":"test-trainjob-tt5jj","reconcileID":"b8f80b71-1f80-461d-b23a-490b9a33870b","job":"test-ns-ljr2h/test-trainjob-tt5jj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ljr2h"}
{"level":"Level(-2)","ts":"2026-11-04T04:15:07.355622427Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ljr2h","name":"test-trainjob-tt5jj","reconcileID":"5e1bbf49-3d07-4241-bb51-7a64b4b7dd69","job":"test-ns-ljr2h/test-trainjob-tt5jj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ljr2h"}
{"level":"Level(-2)","ts":"2026-11-04T04:15:07.468656679Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ljr2h","name":"test-trainjob-tt5jj","reconcileID":"99395828-9912-4bfd-9d80-7dee3b345a45","job":"test-ns-ljr2h/test-trainjob-tt5jj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ljr2h"}
{"level":"Level(-2)","ts":"2026-11-04T04:15:08.394742762Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ljr2h","name":"test-trainjob-tt5jj","reconcileID":"df77eb95-7537-4b18-ad14-88c3f02b4d4e","job":"test-ns-ljr2h/test-trainjob-tt5jj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ljr2h"}
{"level":"Level(-2)","ts":"2026-11-04T04:15:08.509367279Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ljr2h","name":"test-trainjob-tt5jj","reconcileID":"a0227c7e-ad2b-4194-b3f6-150692590062","job":"test-ns-ljr2h/test-trainjob-tt5jj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ljr2h"}
{"level":"Level(-2)","ts":"2026-11-04T04:15:10.590089662Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ljr2h","name":"test-trainjob-tt5jj","reconcileID":"a9e024af-eaa7-4b3a-90c1-8f546da23ee3","job":"test-ns-ljr2h/test-trainjob-tt5jj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ljr2h"}
{"level":"Level(-2)","ts":"2026-11-04T04:15:10.656352462Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ljr2h","name":"test-trainjob-tt5jj","reconcileID":"ccd84add-e648-4b08-bd43-a02c36890b87","job":"test-ns-ljr2h/test-trainjob-tt5jj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ljr2h"}
{"level":"Level(-2)","ts":"2026-11-04T04:15:10.704275681Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ljr2h","name":"test-trainjob-tt5jj","reconcileID":"436737cd-1087-447f-8eff-2344102d917e","job":"test-ns-ljr2h/test-trainjob-tt5jj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ljr2h"}
{"level":"Level(-2)","ts":"2026-11-04T04:15:10.817049012Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ljr2h","name":"test-trainjob-tt5jj","reconcileID":"4a94b796-6a15-4f22-bc82-8084794b38ef","job":"test-ns-ljr2h/test-trainjob-tt5jj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ljr2h"}
{"level":"Level(-2)","ts":"2026-11-04T04:15:12.253387153Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jcdtx","name":"test-trainjob-r824f","reconcileID":"7ae119a4-c8ef-4f68-849a-e392714f32c4","job":"test-ns-jcdtx/test-trainjob-r824f","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jcdtx"}
{"level":"Level(-2)","ts":"2026-11-04T04:15:12.83036603Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jcdtx","name":"test-trainjob-r824f","reconcileID":"2cdd4bda-c366-4b64-aa23-2ffac1ec5dec","job":"test-ns-jcdtx/test-trainjob-r824f","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jcdtx"}
{"level":"Level(-2)","ts":"2026-11-04T04:15:13.065741195Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jcdtx","name":"test-trainjob-r824f","reconcileID":"cf003a29-6ce0-4425-8192-2db80005c620","job":"test-ns-jcdtx/test-trainjob-r824f","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jcdtx"}
{"level":"Level(-2)","ts":"2026-11-04T04:15:13.183161715Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jcdtx","name":"test-trainjob-r824f","reconcileID":"2f15a52a-26c6-4cd3-91ab-a3f3dc6e680e","job":"test-ns-jcdtx/test-trainjob-r824f","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jcdtx"}
{"level":"Level(-2)","ts":"2026-11-04T04:15:13.414623909Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jcdtx","name":"test-trainjob-r824f","reconcileID":"dd816610-b2f8-4d12-9ab9-5ccfd0077c4a","job":"test-ns-jcdtx/test-trainjob-r824f","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jcdtx"}
{"level":"Level(-2)","ts":"2026-11-04T04:15:13.532111883Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jcdtx","name":"test-trainjob-r824f","reconcileID":"8b06aa31-b0d7-4b5f-b6d5-5bda5ee0b6d7","job":"test-ns-jcdtx/test-trainjob-r824f","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jcdtx"}
{"level":"Level(-2)","ts":"2026-11-04T04:15:16.577129173Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ljr2h","name":"test-trainjob-tt5jj","reconcileID":"5ccfb79c-8ea6-434f-828d-3d1886e423f0","job":"test-ns-ljr2h/test-trainjob-tt5jj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ljr2h"}
{"level":"Level(-2)","ts":"2026-11-04T04:15:16.689291194Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ljr2h","name":"test-trainjob-tt5jj","reconcileID":"9543b771-822f-413f-b2ed-4ec710b09fdf","job":"test-ns-ljr2h/test-trainjob-tt5jj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ljr2h"}
{"level":"Level(-2)","ts":"2026-11-04T04:15:17.131902383Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-5d9h2","name":"test-trainjob-dkbxt","reconcileID":"35d8bdf5-71ce-49c9-a1ef-b6c7489864b3","job":"test-ns-5d9h2/test-trainjob-dkbxt","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-5d9h2"}
{"level":"Level(-2)","ts":"2026-11-04T04:15:17.874057272Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ljr2h","name":"test-trainjob-tt5jj","reconcileID":"29a70e0c-3a0c-4d5e-a4db-9525f5933069","job":"test-ns-ljr2h/test-trainjob-tt5jj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ljr2h"}
{"level":"Level(-2)","ts":"2026-11-04T04:15:34.473745877Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jcdtx","name":"test-trainjob-r824f","reconcileID":"00d0b721-e811-4de5-977c-cac228516fdd","job":"test-ns-jcdtx/test-trainjob-r824f","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jcdtx"}
{"level":"Level(-2)","ts":"2026-11-04T04:15:34.591608747Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jcdtx","name":"test-trainjob-r824f","reconcileID":"c75e129c-17d4-476f-8f7a-b235456e4df0","job":"test-ns-jcdtx/test-trainjob-r824f","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jcdtx"}
{"level":"Level(-2)","ts":"2026-11-04T04:15:39.492817531Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jcdtx","name":"test-trainjob-r824f","reconcileID":"20abbe87-d5bb-4bb4-8750-a34b46596b4a","job":"test-ns-jcdtx/test-trainjob-r824f","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jcdtx"}
{"level":"Level(-2)","ts":"2026-11-04T04:15:39.612612571Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jcdtx","name":"test-trainjob-r824f","reconcileID":"82289328-389e-4ff4-8c97-7b9f1f60c17c","job":"test-ns-jcdtx/test-trainjob-r824f","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jcdtx"}
{"level":"Level(-2)","ts":"2026-11-04T04:15:40.501820747Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jcdtx","name":"test-trainjob-r824f","reconcileID":"e3986402-2354-4cc1-b7fb-8170e232eacc","job":"test-ns-jcdtx/test-trainjob-r824f","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jcdtx"}
{"level":"Level(-2)","ts":"2026-11-04T04:15:40.620722539Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jcdtx","name":"test-trainjob-r824f","reconcileID":"6dc689c4-ec82-412b-a81e-077ff8b77f42","job":"test-ns-jcdtx/test-trainjob-r824f","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jcdtx"}
{"level":"Level(-2)","ts":"2026-11-04T04:15:40.734476473Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jcdtx","name":"test-trainjob-r824f","reconcileID":"0e17cab8-42a5-43fb-bbf3-a49738d9f19c","job":"test-ns-jcdtx/test-trainjob-r824f","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jcdtx"}
{"level":"Level(-2)","ts":"2026-11-04T04:15:40.852380513Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jcdtx","name":"test-trainjob-r824f","reconcileID":"68939a2f-3aab-4adf-8d7c-de4bab41bea8","job":"test-ns-jcdtx/test-trainjob-r824f","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jcdtx"}
{"level":"Level(-2)","ts":"2026-11-04T04:15:40.861458025Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jcdtx","name":"test-trainjob-r824f","reconcileID":"2902e635-aa59-4ef9-8688-6110d93f7267","job":"test-ns-jcdtx/test-trainjob-r824f","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jcdtx"}
{"level":"Level(-2)","ts":"2026-11-04T04:15:40.969709045Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jcdtx","name":"test-trainjob-r824f","reconcileID":"85316e98-fff6-4d15-89c9-64b2a4535f36","job":"test-ns-jcdtx/test-trainjob-r824f","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jcdtx"}
{"level":"Level(-2)","ts":"2026-11-04T04:16:54.821454343Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jcdtx","name":"test-trainjob-r824f","reconcileID":"e023d4c4-189d-4ae7-b031-7d54dd87806f","job":"test-ns-jcdtx/test-trainjob-r824f","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jcdtx"}
{"level":"Level(-2)","ts":"2026-11-04T04:16:54.939304486Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jcdtx","name":"test-trainjob-r824f","reconcileID":"9f89d9e6-ed50-4396-a1fc-c59d934695a1","job":"test-ns-jcdtx/test-trainjob-r824f","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jcdtx"}
{"level":"Level(-2)","ts":"2026-11-04T04:17:05.864817624Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jcdtx","name":"test-trainjob-r824f","reconcileID":"d8ab9a43-7c3b-4660-a8d7-2189871f7a36","job":"test-ns-jcdtx/test-trainjob-r824f","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jcdtx"}
{"level":"Level(-2)","ts":"2026-11-04T04:17:05.983056584Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jcdtx","name":"test-trainjob-r824f","reconcileID":"488ef0bd-598a-4550-bc42-da318bd5bafc","job":"test-ns-jcdtx/test-trainjob-r824f","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jcdtx"}
{"level":"Level(-2)","ts":"2026-11-04T04:17:06.874325108Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jcdtx","name":"test-trainjob-r824f","reconcileID":"873c3e3f-8c96-4658-9ba6-2fa537cddf69","job":"test-ns-jcdtx/test-trainjob-r824f","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jcdtx"}
{"level":"Level(-2)","ts":"2026-11-04T04:17:06.992016683Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jcdtx","name":"test-trainjob-r824f","reconcileID":"49c8d29b-7396-41a6-804b-7a9fe3597d4e","job":"test-ns-jcdtx/test-trainjob-r824f","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jcdtx"}
{"level":"Level(-2)","ts":"2026-11-04T04:17:07.107128396Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jcdtx","name":"test-trainjob-r824f","reconcileID":"a4efe131-a8c8-4d27-ab77-7e7f407998d9","job":"test-ns-jcdtx/test-trainjob-r824f","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jcdtx"}
{"level":"Level(-2)","ts":"2026-11-04T04:17:07.224827061Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jcdtx","name":"test-trainjob-r824f","reconcileID":"dbf69bf4-66f0-4950-a6bc-eb1b46a296d8","job":"test-ns-jcdtx/test-trainjob-r824f","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jcdtx"}
{"level":"Level(-2)","ts":"2026-11-04T04:17:07.235832823Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jcdtx","name":"test-trainjob-r824f","reconcileID":"cc7c5973-7b08-4f80-bd6a-f6b44b0ae31e","job":"test-ns-jcdtx/test-trainjob-r824f","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jcdtx"}
{"level":"Level(-2)","ts":"2026-11-04T04:17:07.342830098Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jcdtx","name":"test-trainjob-r824f","reconcileID":"8eb37097-c35e-4ffd-b40d-3be1d8c27378","job":"test-ns-jcdtx/test-trainjob-r824f","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jcdtx"}
{"level":"Level(-2)","ts":"2026-11-04T04:17:12.02602821Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jcdtx","name":"test-trainjob-r824f","reconcileID":"d652e007-e720-422e-bd78-ac2e0c3bad0a","job":"test-ns-jcdtx/test-trainjob-r824f","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jcdtx"}
{"level":"Level(-2)","ts":"2026-11-04T04:17:12.146420106Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jcdtx","name":"test-trainjob-r824f","reconcileID":"e0364d55-aa1c-4385-9b3f-d58b8a3ca32b","job":"test-ns-jcdtx/test-trainjob-r824f","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jcdtx"}
{"level":"Level(-2)","ts":"2026-11-04T04:17:12.190584714Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jcdtx","name":"test-trainjob-r824f","reconcileID":"c3719fa5-5539-4716-87af-728a55165793","job":"test-ns-jcdtx/test-trainjob-r824f","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jcdtx"}
{"level":"Level(-2)","ts":"2026-11-04T04:17:12.263869484Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jcdtx","name":"test-trainjob-r824f","reconcileID":"06235278-7e4e-4261-b971-3ec004fce95e","job":"test-ns-jcdtx/test-trainjob-r824f","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jcdtx"}
{"level":"Level(-2)","ts":"2026-11-04T04:17:19.269791372Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jcdtx","name":"test-trainjob-r824f","reconcileID":"45e865b5-9dd4-41c4-83e4-53de261246f3","job":"test-ns-jcdtx/test-trainjob-r824f","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jcdtx"}
{"level":"Level(-2)","ts":"2026-11-04T04:17:39.644363358Z","logger":"tas-node-reconciler","caller":"tas/node_controller.go:136","msg":"Reconcile Node","replica-role":"leader","name":"ip-10-0-135-81.us-west-2.compute.internal","reconcileID":"f770a6fb-807d-43be-830a-69c4464487ce"}
{"level":"Level(-2)","ts":"2026-11-04T04:22:42.750463778Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-hkzlt","name":"w06fb4129995","reconcileID":"48b74117-c463-4bdf-9f32-d036020a46c3","job":"test-ns-hkzlt/w06fb4129995","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-hkzlt"}
{"level":"Level(-2)","ts":"2026-11-04T04:22:43.230404527Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-hkzlt","name":"w06fb4129995","reconcileID":"7b1adf25-662e-43a6-9458-a5c751a7fad8","job":"test-ns-hkzlt/w06fb4129995","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-hkzlt"}
{"level":"Level(-2)","ts":"2026-11-04T04:22:43.465759235Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-hkzlt","name":"w06fb4129995","reconcileID":"6a9bc1c7-081e-432f-be28-696474145cd7","job":"test-ns-hkzlt/w06fb4129995","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-hkzlt"}
{"level":"Level(-2)","ts":"2026-11-04T04:22:43.583435348Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-hkzlt","name":"w06fb4129995","reconcileID":"67cfe920-4960-4476-b198-07e02e769a4d","job":"test-ns-hkzlt/w06fb4129995","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-hkzlt"}
{"level":"Level(-2)","ts":"2026-11-04T04:22:43.98523868Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-hkzlt","name":"w06fb4129995","reconcileID":"1b6980af-f1db-4e59-8428-28a71ba85b29","job":"test-ns-hkzlt/w06fb4129995","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-hkzlt"}
{"level":"Level(-2)","ts":"2026-11-04T04:22:44.102928974Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-hkzlt","name":"w06fb4129995","reconcileID":"7910353b-fab7-460f-9814-a0ecc711007e","job":"test-ns-hkzlt/w06fb4129995","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-hkzlt"}
{"level":"Level(-2)","ts":"2026-11-04T04:22:45.899431021Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-hkzlt","name":"w06fb4129995","reconcileID":"709619ff-98d3-471e-99b2-7e738c191920","job":"test-ns-hkzlt/w06fb4129995","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-hkzlt"}
{"level":"Level(-2)","ts":"2026-11-04T04:22:46.018094838Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-hkzlt","name":"w06fb4129995","reconcileID":"329d9636-9cc2-4aa6-882e-33b8bdc60754","job":"test-ns-hkzlt/w06fb4129995","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-hkzlt"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:15.522625469Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-hkzlt","name":"w06fb4129995","reconcileID":"ff64e8f0-d757-4837-abec-17c931495bc9","job":"test-ns-hkzlt/w06fb4129995","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-hkzlt"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:15.640209007Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-hkzlt","name":"w06fb4129995","reconcileID":"3fff3d40-65b9-42ad-a4a4-7bf0c0871d11","job":"test-ns-hkzlt/w06fb4129995","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-hkzlt"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:16.532163679Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-hkzlt","name":"w06fb4129995","reconcileID":"d9a2118a-9dad-4be3-aed3-20ad23d23921","job":"test-ns-hkzlt/w06fb4129995","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-hkzlt"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:16.650008995Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-hkzlt","name":"w06fb4129995","reconcileID":"8e65421d-c3a4-47a8-a8b5-1aca2769ba75","job":"test-ns-hkzlt/w06fb4129995","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-hkzlt"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:16.694574227Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-hkzlt","name":"w06fb4129995","reconcileID":"ddee01bd-0d5b-429a-8b24-4a3ae878ca6e","job":"test-ns-hkzlt/w06fb4129995","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-hkzlt"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:16.767279069Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-hkzlt","name":"w06fb4129995","reconcileID":"642033c3-6756-4aac-bcd0-4d6e76847915","job":"test-ns-hkzlt/w06fb4129995","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-hkzlt"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:21.488841729Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:141","msg":"ResourceFlavor create event","replica-role":"leader","resourceFlavor":{"name":"rf-pcwdn"}}
{"level":"Level(-2)","ts":"2026-11-04T04:23:21.488984079Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-pcwdn","reconcileID":"935aa53e-6cf7-4df6-922d-869f3bd88876"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:21.864834094Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:351","msg":"ClusterQueue create event","replica-role":"leader","clusterQueue":{"name":"cq-swpxd"}}
{"level":"Level(-2)","ts":"2026-11-04T04:23:21.865085305Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-swpxd","reconcileID":"c817b0db-818b-4abb-8a62-0036737e4d6a"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:21.883143524Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:240","msg":"LocalQueue create event","replica-role":"leader","localQueue":{"name":"lq-d7rrf","namespace":"test-ns-w5fhp"}}
{"level":"Level(-2)","ts":"2026-11-04T04:23:21.883297764Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-w5fhp","name":"lq-d7rrf","reconcileID":"d87e303e-7ae4-41e9-ac50-44b8d9d591a3"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:21.920209242Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-swpxd"}}
{"level":"Level(-2)","ts":"2026-11-04T04:23:21.920375408Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-swpxd","reconcileID":"54a18d68-1964-4a1c-802e-10132d316299"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:21.937519761Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-w5fhp","name":"lq-d7rrf","reconcileID":"7de0e614-9728-4a34-89de-c164fd75eaa3"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:21.93759233Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-d7rrf","namespace":"test-ns-w5fhp"}}
{"level":"Level(-3)","ts":"2026-11-04T04:23:21.987145883Z","logger":"localqueue-reconciler","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-w5fhp","name":"lq-d7rrf","reconcileID":"7de0e614-9728-4a34-89de-c164fd75eaa3","error":"Operation cannot be fulfilled on localqueues.kueue.x-k8s.io \"lq-d7rrf\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:21.987228963Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-w5fhp","name":"lq-d7rrf","reconcileID":"f890dfdb-f513-4ee0-913f-cf80981903bd"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:22.041584323Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-d7rrf","namespace":"test-ns-w5fhp"}}
{"level":"Level(-2)","ts":"2026-11-04T04:23:22.041660277Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-w5fhp","name":"lq-d7rrf","reconcileID":"9dcca10e-3c88-486f-973d-034a9ecfe273"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:22.73494644Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-w5fhp","name":"dw-deployment-h4b5d-5f87cf994b-hh8n4","reconcileID":"f053d5f6-cf75-4d46-a45d-81c1a3a9755a","job":"test-ns-w5fhp/dw-deployment-h4b5d-5f87cf994b-hh8n4","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:22.785438148Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1.PriorityClass","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"debug","ts":"2026-11-04T04:23:23.250620425Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"dw-deployment-h4b5d-5f87cf994b-hh8n4","namespace":"test-ns-w5fhp"},"kind":"Pod","apiVersion":"v1","type":"Normal","reason":"CreatedWorkload","action":"CreatedWorkload","note":"Created Workload: test-ns-w5fhp/pod-dw-deployment-h4b5d-5f87cf994b-hh8n4-9866a"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:23.296861102Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1276","msg":"Workload create event","replica-role":"leader","workload":{"name":"pod-dw-deployment-h4b5d-5f87cf994b-hh8n4-9866a","namespace":"test-ns-w5fhp"},"queue":"lq-d7rrf","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:23.296892437Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-w5fhp","name":"pod-dw-deployment-h4b5d-5f87cf994b-hh8n4-9866a","reconcileID":"1d7a4e9a-a379-4373-9acf-36a3a2db5165"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:23.296908138Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-w5fhp","name":"dw-deployment-h4b5d-5f87cf994b-hh8n4","reconcileID":"0d07a5dd-a1e2-4851-99fa-7bf3c4a84838","job":"test-ns-w5fhp/dw-deployment-h4b5d-5f87cf994b-hh8n4","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:23.296960118Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-w5fhp","name":"pod-dw-deployment-h4b5d-5f87cf994b-hh8n4-9866a","reconcileID":"1d7a4e9a-a379-4373-9acf-36a3a2db5165","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-04T04:23:23.29711647Z","logger":"scheduler","caller":"scheduler/scheduler.go:308","msg":"Obtained heads","replica-role":"follower","schedulingCycle":1,"headCount":1,"waitDuration":2157.05162972}
{"level":"Level(-2)","ts":"2026-11-04T04:23:23.297153477Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-w5fhp","name":"pod-dw-deployment-h4b5d-5f87cf994b-hh8n4-9866a","reconcileID":"7ac5eb9e-9a31-49a5-8cac-0e65e5a3dfdf"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:23.297154918Z","logger":"scheduler","caller":"scheduler/scheduler.go:323","msg":"Snapshot taken","replica-role":"follower","schedulingCycle":1,"duration":0.000012691}
{"level":"Level(-2)","ts":"2026-11-04T04:23:23.297324484Z","logger":"scheduler","caller":"scheduler/scheduler.go:328","msg":"Nomination done","replica-role":"follower","schedulingCycle":1,"entries":1,"inadmissibleEntries":0,"duration":0.000078799}
{"level":"Level(-2)","ts":"2026-11-04T04:23:23.297346106Z","logger":"scheduler","caller":"scheduler/scheduler.go:406","msg":"Attempting to schedule workload","replica-role":"follower","schedulingCycle":1,"workload":{"name":"pod-dw-deployment-h4b5d-5f87cf994b-hh8n4-9866a","namespace":"test-ns-w5fhp"},"clusterQueue":{"name":"cq-swpxd"}}
{"level":"Level(-2)","ts":"2026-11-04T04:23:23.297430924Z","logger":"scheduler","caller":"scheduler/scheduler.go:1041","msg":"Workload assumed in the cache","replica-role":"follower","schedulingCycle":1,"workload":{"name":"pod-dw-deployment-h4b5d-5f87cf994b-hh8n4-9866a","namespace":"test-ns-w5fhp"},"clusterQueue":{"name":"cq-swpxd"}}
{"level":"Level(-2)","ts":"2026-11-04T04:23:23.297459131Z","logger":"scheduler","caller":"scheduler/scheduler.go:362","msg":"Workload processing done","replica-role":"follower","schedulingCycle":1,"duration":0.000116229}
{"level":"Level(-2)","ts":"2026-11-04T04:23:23.297474582Z","logger":"scheduler","caller":"scheduler/scheduler.go:297","msg":"Scheduling cycle complete","replica-role":"follower","schedulingCycle":1,"duration":2157.051988735}
{"level":"Level(-2)","ts":"2026-11-04T04:23:23.297486989Z","logger":"scheduler","caller":"scheduler/scheduler.go:295","msg":"Scheduling cycle starts","replica-role":"leader","schedulingCycle":2}
{"level":"Level(-2)","ts":"2026-11-04T04:23:23.412698019Z","logger":"scheduler","caller":"scheduler/scheduler.go:1000","msg":"Workload successfully admitted and assigned flavors","replica-role":"follower","schedulingCycle":1,"workload":{"name":"pod-dw-deployment-h4b5d-5f87cf994b-hh8n4-9866a","namespace":"test-ns-w5fhp"},"clusterQueue":{"name":"cq-swpxd"},"assignments":[{"name":"main","flavors":{"cpu":"rf-pcwdn","memory":"rf-pcwdn"},"resourceUsage":{"cpu":"1","memory":"3Gi"},"count":1}]}
{"level":"debug","ts":"2026-11-04T04:23:23.412741408Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"pod-dw-deployment-h4b5d-5f87cf994b-hh8n4-9866a","namespace":"test-ns-w5fhp"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"QuotaReserved","action":"QuotaReserved","note":"Quota reserved in ClusterQueue cq-swpxd, wait time since queued was 0s"}
{"level":"debug","ts":"2026-11-04T04:23:23.412754884Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"pod-dw-deployment-h4b5d-5f87cf994b-hh8n4-9866a","namespace":"test-ns-w5fhp"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"Admitted","action":"Admitted","note":"Admitted by ClusterQueue cq-swpxd, wait time since reservation was 0s"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:23.412869874Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"pod-dw-deployment-h4b5d-5f87cf994b-hh8n4-9866a","namespace":"test-ns-w5fhp"},"queue":"lq-d7rrf","status":"admitted","prevStatus":"pending","clusterQueue":"cq-swpxd"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:23.412893396Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-w5fhp","name":"dw-deployment-h4b5d-5f87cf994b-hh8n4","reconcileID":"82e5209b-24d2-4c6c-acff-118dcfabaabc","job":"test-ns-w5fhp/dw-deployment-h4b5d-5f87cf994b-hh8n4","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:23.412907528Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-w5fhp","name":"pod-dw-deployment-h4b5d-5f87cf994b-hh8n4-9866a","reconcileID":"1cf876cd-8d72-44f3-af90-486df9044e2e"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:23.412960863Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-w5fhp","name":"pod-dw-deployment-h4b5d-5f87cf994b-hh8n4-9866a","reconcileID":"1cf876cd-8d72-44f3-af90-486df9044e2e","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-04T04:23:23.413119149Z","logger":"v1_pod","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-w5fhp","name":"dw-deployment-h4b5d-5f87cf994b-hh8n4","reconcileID":"82e5209b-24d2-4c6c-acff-118dcfabaabc","job":"test-ns-w5fhp/dw-deployment-h4b5d-5f87cf994b-hh8n4","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:23.413140123Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-w5fhp","name":"pod-dw-deployment-h4b5d-5f87cf994b-hh8n4-9866a","reconcileID":"8089412b-ebed-4899-a08e-e71ef2a5b734"}
{"level":"debug","ts":"2026-11-04T04:23:23.532062134Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"dw-deployment-h4b5d-5f87cf994b-hh8n4","namespace":"test-ns-w5fhp"},"kind":"Pod","apiVersion":"v1","type":"Normal","reason":"Started","action":"Started","note":"Admitted by clusterQueue cq-swpxd"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:23.532466097Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-w5fhp","name":"dw-deployment-h4b5d-5f87cf994b-hh8n4","reconcileID":"bb272e30-4a9d-4003-8f9e-27addc27ea5d","job":"test-ns-w5fhp/dw-deployment-h4b5d-5f87cf994b-hh8n4","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:23.539847417Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-w5fhp","name":"dw-deployment-h4b5d-5f87cf994b-hh8n4","reconcileID":"f3edc872-97bb-4bf8-a7c8-b7e2ebe4d9c1","job":"test-ns-w5fhp/dw-deployment-h4b5d-5f87cf994b-hh8n4","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:23.60305552Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-w5fhp","name":"dw-deployment-h4b5d-5f87cf994b-hh8n4","reconcileID":"4533b2fc-0c89-423b-ac84-de4a5756b507","job":"test-ns-w5fhp/dw-deployment-h4b5d-5f87cf994b-hh8n4","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:23.700580806Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-w5fhp","name":"dw-deployment-h4b5d-5f87cf994b-hh8n4","reconcileID":"b88e0cc2-e664-4f57-ab87-0f4fe2c08314","job":"test-ns-w5fhp/dw-deployment-h4b5d-5f87cf994b-hh8n4","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:24.297954246Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-w5fhp","name":"lq-d7rrf","reconcileID":"c2f3a8d0-bec8-4bba-9fb1-f5d8e9381ef8"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:24.297956151Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-swpxd","reconcileID":"1990337d-af06-4107-8069-e43b0638c072"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:24.350713816Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-d7rrf","namespace":"test-ns-w5fhp"}}
{"level":"Level(-2)","ts":"2026-11-04T04:23:24.350827283Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-w5fhp","name":"lq-d7rrf","reconcileID":"1c1f51b6-1952-4ece-83a7-f5b838a93d32"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:24.353723807Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-swpxd"}}
{"level":"Level(-2)","ts":"2026-11-04T04:23:24.396470343Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-swpxd","reconcileID":"a71ee35a-26d4-465d-beb4-a44925d74398"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:24.523982306Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-w5fhp","name":"dw-deployment-h4b5d-5f87cf994b-hh8n4","reconcileID":"e68a6355-56f1-4b96-b7f1-58d77206e568","job":"test-ns-w5fhp/dw-deployment-h4b5d-5f87cf994b-hh8n4","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:24.532878969Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-w5fhp","name":"pod-dw-deployment-h4b5d-5f87cf994b-hh8n4-9866a","reconcileID":"c6966b3c-7fc8-4569-8a8f-9e4070e00f3b"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:26.495790722Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-w5fhp","name":"dw-deployment-h4b5d-5f87cf994b-hh8n4","reconcileID":"65f97b51-41c9-44bd-be7a-9e9c39ac6781","job":"test-ns-w5fhp/dw-deployment-h4b5d-5f87cf994b-hh8n4","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:27.496817005Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-w5fhp","name":"pod-dw-deployment-h4b5d-5f87cf994b-hh8n4-9866a","reconcileID":"fae5c1bc-43e6-4104-9410-c233a8ab63bf"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:38.088678707Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-w5fhp","name":"ycbd600a3ac3","reconcileID":"e764e047-4cd2-4b02-a72f-86e6a8fb977f","job":"test-ns-w5fhp/ycbd600a3ac3","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:38.41755771Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1276","msg":"Workload create event","replica-role":"leader","workload":{"name":"trainjob-ycbd600a3ac3-991e3","namespace":"test-ns-w5fhp"},"queue":"lq-d7rrf","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:38.417600777Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-w5fhp","name":"ycbd600a3ac3","reconcileID":"021c0e9d-accf-437b-86f1-a5201e47b81e","job":"test-ns-w5fhp/ycbd600a3ac3","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"debug","ts":"2026-11-04T04:23:38.41762721Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"ycbd600a3ac3","namespace":"test-ns-w5fhp"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"CreatedWorkload","action":"CreatedWorkload","note":"Created Workload: test-ns-w5fhp/trainjob-ycbd600a3ac3-991e3"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:38.417635242Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-w5fhp","name":"trainjob-ycbd600a3ac3-991e3","reconcileID":"82f11667-ccee-4649-8f5d-6a92054ec261"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:38.417683389Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-w5fhp","name":"trainjob-ycbd600a3ac3-991e3","reconcileID":"82f11667-ccee-4649-8f5d-6a92054ec261","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-04T04:23:38.417763258Z","logger":"scheduler","caller":"scheduler/scheduler.go:308","msg":"Obtained heads","replica-role":"leader","schedulingCycle":2,"headCount":1,"waitDuration":15.120275426}
{"level":"Level(-2)","ts":"2026-11-04T04:23:38.417802694Z","logger":"scheduler","caller":"scheduler/scheduler.go:323","msg":"Snapshot taken","replica-role":"leader","schedulingCycle":2,"duration":0.00001385}
{"level":"Level(-2)","ts":"2026-11-04T04:23:38.417827887Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-w5fhp","name":"trainjob-ycbd600a3ac3-991e3","reconcileID":"e52c03a5-cc0d-4473-abc7-be307136d0af"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:38.417895402Z","logger":"scheduler","caller":"scheduler/scheduler.go:328","msg":"Nomination done","replica-role":"leader","schedulingCycle":2,"entries":1,"inadmissibleEntries":0,"duration":0.000075176}
{"level":"Level(-2)","ts":"2026-11-04T04:23:38.417918042Z","logger":"scheduler","caller":"scheduler/scheduler.go:406","msg":"Attempting to schedule workload","replica-role":"leader","schedulingCycle":2,"workload":{"name":"trainjob-ycbd600a3ac3-991e3","namespace":"test-ns-w5fhp"},"clusterQueue":{"name":"cq-swpxd"}}
{"level":"Level(-2)","ts":"2026-11-04T04:23:38.418001192Z","logger":"scheduler","caller":"scheduler/scheduler.go:1041","msg":"Workload assumed in the cache","replica-role":"leader","schedulingCycle":2,"workload":{"name":"trainjob-ycbd600a3ac3-991e3","namespace":"test-ns-w5fhp"},"clusterQueue":{"name":"cq-swpxd"}}
{"level":"Level(-2)","ts":"2026-11-04T04:23:38.418029072Z","logger":"scheduler","caller":"scheduler/scheduler.go:362","msg":"Workload processing done","replica-role":"leader","schedulingCycle":2,"duration":0.00011637}
{"level":"Level(-2)","ts":"2026-11-04T04:23:38.418047903Z","logger":"scheduler","caller":"scheduler/scheduler.go:297","msg":"Scheduling cycle complete","replica-role":"leader","schedulingCycle":2,"duration":15.120560382}
{"level":"Level(-2)","ts":"2026-11-04T04:23:38.418069224Z","logger":"scheduler","caller":"scheduler/scheduler.go:295","msg":"Scheduling cycle starts","replica-role":"leader","schedulingCycle":3}
{"level":"Level(-2)","ts":"2026-11-04T04:23:38.529696978Z","logger":"scheduler","caller":"scheduler/scheduler.go:1000","msg":"Workload successfully admitted and assigned flavors","replica-role":"leader","schedulingCycle":2,"workload":{"name":"trainjob-ycbd600a3ac3-991e3","namespace":"test-ns-w5fhp"},"clusterQueue":{"name":"cq-swpxd"},"assignments":[{"name":"node","flavors":{"cpu":"rf-pcwdn","memory":"rf-pcwdn"},"resourceUsage":{"cpu":"4","memory":"16Gi"},"count":2}]}
{"level":"debug","ts":"2026-11-04T04:23:38.529748808Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-ycbd600a3ac3-991e3","namespace":"test-ns-w5fhp"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"Admitted","action":"Admitted","note":"Admitted by ClusterQueue cq-swpxd, wait time since reservation was 1s"}
{"level":"debug","ts":"2026-11-04T04:23:38.529767933Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-ycbd600a3ac3-991e3","namespace":"test-ns-w5fhp"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"QuotaReserved","action":"QuotaReserved","note":"Quota reserved in ClusterQueue cq-swpxd, wait time since queued was 1s"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:38.529994756Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-ycbd600a3ac3-991e3","namespace":"test-ns-w5fhp"},"queue":"lq-d7rrf","status":"admitted","prevStatus":"pending","clusterQueue":"cq-swpxd"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:38.53005695Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-w5fhp","name":"trainjob-ycbd600a3ac3-991e3","reconcileID":"88320ce9-cb7e-43fe-b99f-c1c71641b105"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:38.530109451Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-w5fhp","name":"trainjob-ycbd600a3ac3-991e3","reconcileID":"88320ce9-cb7e-43fe-b99f-c1c71641b105","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-04T04:23:38.530291433Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-w5fhp","name":"trainjob-ycbd600a3ac3-991e3","reconcileID":"3fd32332-b538-4957-ad42-8053456b6098"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:38.579213968Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-w5fhp","name":"ycbd600a3ac3","reconcileID":"c1657e2c-85e4-4237-8c23-1805c720f3f7","job":"test-ns-w5fhp/ycbd600a3ac3","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:38.739770832Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-w5fhp","name":"ycbd600a3ac3","reconcileID":"c1657e2c-85e4-4237-8c23-1805c720f3f7","job":"test-ns-w5fhp/ycbd600a3ac3","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-3)","ts":"2026-11-04T04:23:38.982105234Z","logger":"trainjob","caller":"jobframework/reconciler.go:594","msg":"Unsuspending job","replica-role":"leader","namespace":"test-ns-w5fhp","name":"ycbd600a3ac3","reconcileID":"c1657e2c-85e4-4237-8c23-1805c720f3f7","job":"test-ns-w5fhp/ycbd600a3ac3","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"ycbd600a3ac3\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-3)","ts":"2026-11-04T04:23:38.982204924Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-w5fhp","name":"ycbd600a3ac3","reconcileID":"c1657e2c-85e4-4237-8c23-1805c720f3f7","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"ycbd600a3ac3\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:38.982323746Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-w5fhp","name":"ycbd600a3ac3","reconcileID":"c1095ad8-f454-48e3-84cb-631646d23fb6","job":"test-ns-w5fhp/ycbd600a3ac3","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:39.143722532Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-w5fhp","name":"ycbd600a3ac3","reconcileID":"c1095ad8-f454-48e3-84cb-631646d23fb6","job":"test-ns-w5fhp/ycbd600a3ac3","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:39.418645432Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-w5fhp","name":"lq-d7rrf","reconcileID":"1bec8b3c-9158-4beb-9cdd-b806837d78e5"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:39.418650506Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-swpxd","reconcileID":"d5018d5a-bea6-4d85-b51d-8a2c28609660"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:39.471490955Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-d7rrf","namespace":"test-ns-w5fhp"}}
{"level":"Level(-2)","ts":"2026-11-04T04:23:39.47161457Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-w5fhp","name":"lq-d7rrf","reconcileID":"9bd14631-af31-46f0-b090-36d102572eb9"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:39.474582288Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-swpxd"}}
{"level":"Level(-2)","ts":"2026-11-04T04:23:39.474733531Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-swpxd","reconcileID":"66a02db6-ec23-4c27-9085-a0a58061a09f"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:39.497708225Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-w5fhp","name":"ycbd600a3ac3","reconcileID":"4fd793b3-9c26-4629-a963-24f33da6a37a","job":"test-ns-w5fhp/ycbd600a3ac3","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"debug","ts":"2026-11-04T04:23:39.497720281Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"ycbd600a3ac3","namespace":"test-ns-w5fhp"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"Started","action":"Started","note":"Admitted by clusterQueue cq-swpxd"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:39.659024968Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-w5fhp","name":"ycbd600a3ac3","reconcileID":"4fd793b3-9c26-4629-a963-24f33da6a37a","job":"test-ns-w5fhp/ycbd600a3ac3","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-3)","ts":"2026-11-04T04:23:39.770013193Z","logger":"trainjob","caller":"jobframework/reconciler.go:594","msg":"Unsuspending job","replica-role":"leader","namespace":"test-ns-w5fhp","name":"ycbd600a3ac3","reconcileID":"4fd793b3-9c26-4629-a963-24f33da6a37a","job":"test-ns-w5fhp/ycbd600a3ac3","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"ycbd600a3ac3\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-3)","ts":"2026-11-04T04:23:39.77009553Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-w5fhp","name":"ycbd600a3ac3","reconcileID":"4fd793b3-9c26-4629-a963-24f33da6a37a","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"ycbd600a3ac3\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:39.770227015Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-w5fhp","name":"ycbd600a3ac3","reconcileID":"84915f7f-2b6d-4fc4-8bb5-f20667215874","job":"test-ns-w5fhp/ycbd600a3ac3","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:39.931514106Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-w5fhp","name":"ycbd600a3ac3","reconcileID":"a2d75ef4-c20f-4acb-93be-cd5d7ab1f5ec","job":"test-ns-w5fhp/ycbd600a3ac3","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:40.024414868Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1.PartialObjectMetadata","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:40.10618127Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-w5fhp","name":"ycbd600a3ac3","reconcileID":"93f02f54-d053-4bcd-b3fc-f921e7921907","job":"test-ns-w5fhp/ycbd600a3ac3","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:40.12486511Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1.PartialObjectMetadata","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:40.26715123Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-w5fhp","name":"ycbd600a3ac3","reconcileID":"50a7177c-695f-43c3-bb8d-2a63aec15ed5","job":"test-ns-w5fhp/ycbd600a3ac3","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:40.428181624Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-w5fhp","name":"ycbd600a3ac3","reconcileID":"5ee7e679-3e4b-43ac-83fa-9b015609b577","job":"test-ns-w5fhp/ycbd600a3ac3","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:40.589398886Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-w5fhp","name":"ycbd600a3ac3","reconcileID":"9407b2ed-19fc-4cd2-bf76-2dad45707913","job":"test-ns-w5fhp/ycbd600a3ac3","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:40.750361353Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-w5fhp","name":"ycbd600a3ac3","reconcileID":"4500322d-d6cb-4204-9494-b4fdf2516aa9","job":"test-ns-w5fhp/ycbd600a3ac3","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:41.293721944Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-w5fhp","name":"trainjob-ycbd600a3ac3-991e3","reconcileID":"69b659db-7ed0-4137-81b9-db5e95193760"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:42.386653764Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-w5fhp","name":"trainjob-ycbd600a3ac3-991e3","reconcileID":"70c299d4-bbd3-4ff2-9fd3-ffdc9ed2ce7d"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:43.61710059Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-w5fhp","name":"trainjob-ycbd600a3ac3-991e3","reconcileID":"91fad3ce-b34e-4a37-8cfe-c2bc08df6d1f"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:43.693056787Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-w5fhp","name":"ycbd600a3ac3","reconcileID":"05d5f441-8682-41df-92df-a3febcf1ce75","job":"test-ns-w5fhp/ycbd600a3ac3","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:43.855985067Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-w5fhp","name":"ycbd600a3ac3","reconcileID":"5eb20711-309b-4c40-993e-e95c22774400","job":"test-ns-w5fhp/ycbd600a3ac3","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:12.851670518Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-w5fhp","name":"trainjob-ycbd600a3ac3-991e3","reconcileID":"90d114d4-2c08-477c-8254-51c3581ee3d2"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:12.87928556Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-w5fhp","name":"ycbd600a3ac3","reconcileID":"834cddc0-494d-48c6-a5fc-a796507e2829","job":"test-ns-w5fhp/ycbd600a3ac3","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:13.040769935Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-w5fhp","name":"ycbd600a3ac3","reconcileID":"97730b49-2c74-4a15-a02d-1036272c9f48","job":"test-ns-w5fhp/ycbd600a3ac3","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:13.888181562Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-w5fhp","name":"ycbd600a3ac3","reconcileID":"9636528d-3802-45f9-bf6f-32756d815eca","job":"test-ns-w5fhp/ycbd600a3ac3","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:13.95981295Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-w5fhp","name":"trainjob-ycbd600a3ac3-991e3","reconcileID":"ea6e7c96-2325-4f75-8be5-16aaf7c253db"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:14.049521258Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-w5fhp","name":"ycbd600a3ac3","reconcileID":"6c743628-2740-4b05-9c3d-e0b00af2375e","job":"test-ns-w5fhp/ycbd600a3ac3","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:14.210755654Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-w5fhp","name":"ycbd600a3ac3","reconcileID":"5279cc1d-494f-44fb-8085-f1c0b5eb5b0b","job":"test-ns-w5fhp/ycbd600a3ac3","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"debug","ts":"2026-11-04T04:24:14.485890011Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"ycbd600a3ac3","namespace":"test-ns-w5fhp"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"FinishedWorkload","action":"FinishedWorkload","note":"Workload 'test-ns-w5fhp/trainjob-ycbd600a3ac3-991e3' is declared finished"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:14.48616455Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-ycbd600a3ac3-991e3","namespace":"test-ns-w5fhp"},"queue":"lq-d7rrf","status":"finished","prevStatus":"admitted","clusterQueue":"cq-swpxd"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:14.486204217Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-w5fhp","name":"trainjob-ycbd600a3ac3-991e3","reconcileID":"5e3cde0b-374d-473f-a485-d072e3e1249d"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:14.486253429Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-w5fhp","name":"trainjob-ycbd600a3ac3-991e3","reconcileID":"5e3cde0b-374d-473f-a485-d072e3e1249d","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-04T04:24:14.486343097Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-w5fhp","name":"ycbd600a3ac3","reconcileID":"34646b0f-728a-4a57-a47b-97855a3f9c3d","job":"test-ns-w5fhp/ycbd600a3ac3","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:14.486397474Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-w5fhp","name":"trainjob-ycbd600a3ac3-991e3","reconcileID":"88aff758-2bf4-47d2-8c2e-9e0b0d135617"}
{"level":"debug","ts":"2026-11-04T04:24:14.65961548Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"ycbd600a3ac3","namespace":"test-ns-w5fhp"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"FinishedWorkload","action":"FinishedWorkload","note":"Workload 'test-ns-w5fhp/trainjob-ycbd600a3ac3-991e3' is declared finished"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:14.7769971Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-ycbd600a3ac3-991e3","namespace":"test-ns-w5fhp"},"queue":"lq-d7rrf","status":"finished","clusterQueue":"cq-swpxd"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:14.777070204Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-w5fhp","name":"trainjob-ycbd600a3ac3-991e3","reconcileID":"47904fbc-0e72-424d-9dd1-a19935ce24ca"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:14.777124846Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-w5fhp","name":"trainjob-ycbd600a3ac3-991e3","reconcileID":"47904fbc-0e72-424d-9dd1-a19935ce24ca","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-04T04:24:14.777174733Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-w5fhp","name":"ycbd600a3ac3","reconcileID":"0cbeb50e-4336-46bd-96f5-93a36107b413","job":"test-ns-w5fhp/ycbd600a3ac3","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:14.777216215Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-w5fhp","name":"trainjob-ycbd600a3ac3-991e3","reconcileID":"b918ae98-4db0-445d-a64f-401136d67d95"}
{"level":"debug","ts":"2026-11-04T04:24:14.938450811Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"ycbd600a3ac3","namespace":"test-ns-w5fhp"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"FinishedWorkload","action":"FinishedWorkload","note":"Workload 'test-ns-w5fhp/trainjob-ycbd600a3ac3-991e3' is declared finished"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:14.980415552Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-w5fhp","name":"trainjob-ycbd600a3ac3-991e3","reconcileID":"53cb92e0-d261-4114-a820-9f52db42db6d"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:15.487496879Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-w5fhp","name":"lq-d7rrf","reconcileID":"72cf1bc3-1bda-4bcc-bb4b-7a89458ab665"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:15.487516021Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-swpxd","reconcileID":"7247f98c-3e21-45fe-82dc-2d00a06678f6"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:15.541765956Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-d7rrf","namespace":"test-ns-w5fhp"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:15.541894063Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-w5fhp","name":"lq-d7rrf","reconcileID":"64c55143-6ce0-4706-bae6-fe15f2e66301"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:15.544718688Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-swpxd"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:15.544856679Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-swpxd","reconcileID":"e0b35dab-a9c2-4e6c-8045-8b65b113cd40"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:16.347687546Z","logger":"trainjob","caller":"workload/workload.go:1810","msg":"Workload is orphaned; finishing to release quota","replica-role":"leader","namespace":"test-ns-w5fhp","name":"ycbd600a3ac3","reconcileID":"21330687-7ef0-40c3-b207-742d1cd11530","job":"test-ns-w5fhp/ycbd600a3ac3","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:16.394024651Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1312","msg":"Workload delete event","replica-role":"leader","workload":{"name":"trainjob-ycbd600a3ac3-991e3","namespace":"test-ns-w5fhp"},"queue":"lq-d7rrf","status":"finished"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:16.394085243Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-w5fhp","name":"trainjob-ycbd600a3ac3-991e3","reconcileID":"3bd55fe3-e43f-40b9-a02a-b1c83a5d32af"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:16.394129731Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-w5fhp","name":"trainjob-ycbd600a3ac3-991e3","reconcileID":"3bd55fe3-e43f-40b9-a02a-b1c83a5d32af","isDeleted":true}
{"level":"Level(-2)","ts":"2026-11-04T04:24:17.395019307Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-swpxd","reconcileID":"d478ed25-2cb7-40b1-9c88-4231aca9b50a"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:17.395015852Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-w5fhp","name":"lq-d7rrf","reconcileID":"7b7ae9b5-707a-43b8-aa22-b9216f1659d9"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:18.170407351Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-swpxd"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:18.170503354Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-swpxd","reconcileID":"8ab77bc7-2442-43ef-b781-6ac2e4a5ffb2"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:18.170637612Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-w5fhp","name":"pod-dw-deployment-h4b5d-5f87cf994b-hh8n4-9866a","reconcileID":"3cc65b74-043a-426d-90b4-11dd1c8135b6"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:18.187851005Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:171","msg":"ResourceFlavor update event","replica-role":"leader","resourceFlavor":{"name":"rf-pcwdn"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:18.187950605Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-pcwdn","reconcileID":"ba4c09ae-7182-4e7c-a353-305884c85f8e"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:18.231475505Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-w5fhp","name":"lq-d7rrf","reconcileID":"fe4bc53e-d9e1-4cfd-b3d5-553b15d8c3d3"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:18.231490151Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-swpxd"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:18.231563767Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-swpxd","reconcileID":"f6da1576-b886-4a2d-8837-85b41dd7fb37"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:18.231633414Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-w5fhp","name":"pod-dw-deployment-h4b5d-5f87cf994b-hh8n4-9866a","reconcileID":"ce04db9f-4b74-4a6f-ab7b-6690873da164"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:18.286206608Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-d7rrf","namespace":"test-ns-w5fhp"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:18.286362025Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-w5fhp","name":"lq-d7rrf","reconcileID":"883ec911-c653-4655-9e07-317ad106e8e3"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:18.520584772Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:141","msg":"ResourceFlavor create event","replica-role":"leader","resourceFlavor":{"name":"rf-rt7ms"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:18.520673279Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-rt7ms","reconcileID":"45fc2d86-8e48-41f9-85f9-0ed4a1e49eaa"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:18.647722198Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:351","msg":"ClusterQueue create event","replica-role":"leader","clusterQueue":{"name":"cq-c22x6"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:18.647921966Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-c22x6","reconcileID":"77d84257-7ffe-4548-9de4-6c61b716c5be"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:18.66458492Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:240","msg":"LocalQueue create event","replica-role":"leader","localQueue":{"name":"lq-bl6f8","namespace":"test-ns-dqfgt"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:18.664717833Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-dqfgt","name":"lq-bl6f8","reconcileID":"cc9132e1-d9b7-48ca-a201-ac6def600df7"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:18.700726342Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-c22x6"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:18.700911264Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-c22x6","reconcileID":"f6623931-3306-400a-8687-9eafd0c723b4"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:18.716920552Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-bl6f8","namespace":"test-ns-dqfgt"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:18.716946872Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-dqfgt","name":"lq-bl6f8","reconcileID":"93f98d61-8c5e-471b-bbdd-8b0a813a8632"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:18.771119873Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-dqfgt","name":"lq-bl6f8","reconcileID":"71a7e6a7-7e2c-4ca7-be85-0fef42e4a8ca"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:18.771124999Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-bl6f8","namespace":"test-ns-dqfgt"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:18.771254273Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-dqfgt","name":"lq-bl6f8","reconcileID":"5d4fa2df-1cd3-4887-a2e4-9315c6cdcd8f"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:18.851051255Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-dqfgt","name":"test-preemption-trainjob-t7bj6","reconcileID":"092447a6-4aef-42ed-b164-7589765555e8","job":"test-ns-dqfgt/test-preemption-trainjob-t7bj6","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:19.153636122Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-w5fhp","name":"pod-dw-deployment-h4b5d-5f87cf994b-hh8n4-9866a","reconcileID":"be3fe564-3dba-49c3-b6bc-25d0d265aa20"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:19.171468338Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-dqfgt","name":"test-preemption-trainjob-t7bj6","reconcileID":"87b45b2f-b7b6-419b-ac11-6c38932b6a13","job":"test-ns-dqfgt/test-preemption-trainjob-t7bj6","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:19.171477896Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1276","msg":"Workload create event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-t7bj6-a698f","namespace":"test-ns-dqfgt"},"queue":"lq-bl6f8","status":"pending"}
{"level":"debug","ts":"2026-11-04T04:24:19.171491261Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"test-preemption-trainjob-t7bj6","namespace":"test-ns-dqfgt"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"CreatedWorkload","action":"CreatedWorkload","note":"Created Workload: test-ns-dqfgt/trainjob-test-preemption-trainjob-t7bj6-a698f"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:19.17159402Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-dqfgt","name":"trainjob-test-preemption-trainjob-t7bj6-a698f","reconcileID":"cec40fe0-f58f-47f3-a628-c6ca89e1249c"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:19.171647195Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-dqfgt","name":"trainjob-test-preemption-trainjob-t7bj6-a698f","reconcileID":"cec40fe0-f58f-47f3-a628-c6ca89e1249c","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-04T04:24:19.171657165Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-dqfgt","name":"trainjob-test-preemption-trainjob-t7bj6-a698f","reconcileID":"83e943bd-0a7a-42de-9c12-bd556dcaaecd"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:19.171684497Z","logger":"scheduler","caller":"scheduler/scheduler.go:308","msg":"Obtained heads","replica-role":"leader","schedulingCycle":3,"headCount":1,"waitDuration":40.753614424}
{"level":"Level(-2)","ts":"2026-11-04T04:24:19.171802124Z","logger":"scheduler","caller":"scheduler/scheduler.go:323","msg":"Snapshot taken","replica-role":"leader","schedulingCycle":3,"duration":0.000012058}
{"level":"Level(-2)","ts":"2026-11-04T04:24:19.171865609Z","logger":"scheduler","caller":"scheduler/scheduler.go:328","msg":"Nomination done","replica-role":"leader","schedulingCycle":3,"entries":1,"inadmissibleEntries":0,"duration":0.000048984}
{"level":"Level(-2)","ts":"2026-11-04T04:24:19.171882407Z","logger":"scheduler","caller":"scheduler/scheduler.go:406","msg":"Attempting to schedule workload","replica-role":"leader","schedulingCycle":3,"workload":{"name":"trainjob-test-preemption-trainjob-t7bj6-a698f","namespace":"test-ns-dqfgt"},"clusterQueue":{"name":"cq-c22x6"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:19.171939343Z","logger":"scheduler","caller":"scheduler/scheduler.go:1041","msg":"Workload assumed in the cache","replica-role":"leader","schedulingCycle":3,"workload":{"name":"trainjob-test-preemption-trainjob-t7bj6-a698f","namespace":"test-ns-dqfgt"},"clusterQueue":{"name":"cq-c22x6"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:19.171957547Z","logger":"scheduler","caller":"scheduler/scheduler.go:362","msg":"Workload processing done","replica-role":"leader","schedulingCycle":3,"duration":0.000078429}
{"level":"Level(-2)","ts":"2026-11-04T04:24:19.171969676Z","logger":"scheduler","caller":"scheduler/scheduler.go:297","msg":"Scheduling cycle complete","replica-role":"leader","schedulingCycle":3,"duration":40.75390037}
{"level":"Level(-2)","ts":"2026-11-04T04:24:19.171982031Z","logger":"scheduler","caller":"scheduler/scheduler.go:295","msg":"Scheduling cycle starts","replica-role":"leader","schedulingCycle":4}
{"level":"Level(-2)","ts":"2026-11-04T04:24:19.28323541Z","logger":"scheduler","caller":"scheduler/scheduler.go:1000","msg":"Workload successfully admitted and assigned flavors","replica-role":"leader","schedulingCycle":3,"workload":{"name":"trainjob-test-preemption-trainjob-t7bj6-a698f","namespace":"test-ns-dqfgt"},"clusterQueue":{"name":"cq-c22x6"},"assignments":[{"name":"node","count":1}]}
{"level":"debug","ts":"2026-11-04T04:24:19.283272454Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-test-preemption-trainjob-t7bj6-a698f","namespace":"test-ns-dqfgt"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"Admitted","action":"Admitted","note":"Admitted by ClusterQueue cq-c22x6, wait time since reservation was 0s"}
{"level":"debug","ts":"2026-11-04T04:24:19.283284Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-test-preemption-trainjob-t7bj6-a698f","namespace":"test-ns-dqfgt"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"QuotaReserved","action":"QuotaReserved","note":"Quota reserved in ClusterQueue cq-c22x6, wait time since queued was 0s"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:19.283337148Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-dqfgt","name":"trainjob-test-preemption-trainjob-t7bj6-a698f","reconcileID":"f1d45a0c-42ce-48bf-b104-c40940f9d3b2"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:19.283382094Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-dqfgt","name":"trainjob-test-preemption-trainjob-t7bj6-a698f","reconcileID":"f1d45a0c-42ce-48bf-b104-c40940f9d3b2","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-04T04:24:19.283332263Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-t7bj6-a698f","namespace":"test-ns-dqfgt"},"queue":"lq-bl6f8","status":"admitted","prevStatus":"pending","clusterQueue":"cq-c22x6"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:19.283531945Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-dqfgt","name":"trainjob-test-preemption-trainjob-t7bj6-a698f","reconcileID":"ad58c5a4-1161-4c4f-b785-c0ae15c9a237"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:19.328462266Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-dqfgt","name":"test-preemption-trainjob-t7bj6","reconcileID":"817e5047-4b7f-4eb0-8a49-ebf3a4070a8c","job":"test-ns-dqfgt/test-preemption-trainjob-t7bj6","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:19.48752142Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-dqfgt","name":"test-preemption-trainjob-t7bj6","reconcileID":"817e5047-4b7f-4eb0-8a49-ebf3a4070a8c","job":"test-ns-dqfgt/test-preemption-trainjob-t7bj6","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-3)","ts":"2026-11-04T04:24:19.596223848Z","logger":"trainjob","caller":"jobframework/reconciler.go:594","msg":"Unsuspending job","replica-role":"leader","namespace":"test-ns-dqfgt","name":"test-preemption-trainjob-t7bj6","reconcileID":"817e5047-4b7f-4eb0-8a49-ebf3a4070a8c","job":"test-ns-dqfgt/test-preemption-trainjob-t7bj6","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-preemption-trainjob-t7bj6\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-3)","ts":"2026-11-04T04:24:19.596302026Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-dqfgt","name":"test-preemption-trainjob-t7bj6","reconcileID":"817e5047-4b7f-4eb0-8a49-ebf3a4070a8c","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-preemption-trainjob-t7bj6\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:19.596415584Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-dqfgt","name":"test-preemption-trainjob-t7bj6","reconcileID":"56f5db14-0cf6-4def-a213-1d144ec4cb0e","job":"test-ns-dqfgt/test-preemption-trainjob-t7bj6","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:19.754450034Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-dqfgt","name":"test-preemption-trainjob-t7bj6","reconcileID":"56f5db14-0cf6-4def-a213-1d144ec4cb0e","job":"test-ns-dqfgt/test-preemption-trainjob-t7bj6","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"debug","ts":"2026-11-04T04:24:20.09362834Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"test-preemption-trainjob-t7bj6","namespace":"test-ns-dqfgt"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"Started","action":"Started","note":"Admitted by clusterQueue cq-c22x6"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:20.093646016Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-dqfgt","name":"test-preemption-trainjob-t7bj6","reconcileID":"06d06903-70d1-4692-9b3c-b22af6e15f7d","job":"test-ns-dqfgt/test-preemption-trainjob-t7bj6","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:20.171760769Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-dqfgt","name":"lq-bl6f8","reconcileID":"21475255-1223-4551-9fac-cbd6e49ca99b"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:20.171769582Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-c22x6","reconcileID":"aad6aa94-25f8-49a1-b6aa-a03f6250e45c"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:20.22665822Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-bl6f8","namespace":"test-ns-dqfgt"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:20.226756885Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-dqfgt","name":"lq-bl6f8","reconcileID":"a35dc914-ad4d-4885-bafc-cc37eb940b19"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:20.226870385Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-c22x6"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:20.227006846Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-c22x6","reconcileID":"cf3741c5-e1f2-490b-949e-329fec85d2fc"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:20.251718683Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-dqfgt","name":"test-preemption-trainjob-t7bj6","reconcileID":"76e4899f-42a4-4b15-8cba-a11ab47ffab2","job":"test-ns-dqfgt/test-preemption-trainjob-t7bj6","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:20.409976871Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-dqfgt","name":"test-preemption-trainjob-t7bj6","reconcileID":"67a15790-0161-4de1-b910-b74a8bc09831","job":"test-ns-dqfgt/test-preemption-trainjob-t7bj6","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:20.568282271Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-dqfgt","name":"test-preemption-trainjob-t7bj6","reconcileID":"307cc234-6c46-4dbc-a78a-0b0a13f2b0f6","job":"test-ns-dqfgt/test-preemption-trainjob-t7bj6","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:20.726215942Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-dqfgt","name":"test-preemption-trainjob-t7bj6","reconcileID":"8fca37c7-4fd1-48ee-9805-a0e98e41e4b3","job":"test-ns-dqfgt/test-preemption-trainjob-t7bj6","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:21.460789754Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-dqfgt","name":"trainjob-test-preemption-trainjob-t7bj6-a698f","reconcileID":"37436d19-90ce-4e49-8209-46dfc5757e55"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:22.011361708Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-t7bj6-a698f","namespace":"test-ns-dqfgt"},"queue":"lq-bl6f8","status":"admitted","clusterQueue":"cq-c22x6"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:22.011408928Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-dqfgt","name":"trainjob-test-preemption-trainjob-t7bj6-a698f","reconcileID":"5b6d1c23-d0c3-4ab4-acf6-6ebd86b824d2"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:22.011437253Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1381","msg":"Workload will not be queued because the workload is not active","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-t7bj6-a698f","namespace":"test-ns-dqfgt"},"queue":"lq-bl6f8","status":"admitted","clusterQueue":"cq-c22x6"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:22.011471017Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-dqfgt","name":"trainjob-test-preemption-trainjob-t7bj6-a698f","reconcileID":"5b6d1c23-d0c3-4ab4-acf6-6ebd86b824d2","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-04T04:24:22.011490324Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-dqfgt","name":"test-preemption-trainjob-t7bj6","reconcileID":"a08fb3e1-96c9-4a2b-9514-e071c850ef6b","job":"test-ns-dqfgt/test-preemption-trainjob-t7bj6","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:22.011560747Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-dqfgt","name":"trainjob-test-preemption-trainjob-t7bj6-a698f","reconcileID":"428a7177-f03f-400f-b6ff-92d9e41eafa3"}
{"level":"debug","ts":"2026-11-04T04:24:22.122306026Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-test-preemption-trainjob-t7bj6-a698f","namespace":"test-ns-dqfgt"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"EvictedDueToDeactivated","action":"EvictedDueToDeactivated","note":"The workload is deactivated"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:22.122409087Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-t7bj6-a698f","namespace":"test-ns-dqfgt"},"queue":"lq-bl6f8","status":"admitted","clusterQueue":"cq-c22x6"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:22.122460358Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1381","msg":"Workload will not be queued because the workload is not active","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-t7bj6-a698f","namespace":"test-ns-dqfgt"},"queue":"lq-bl6f8","status":"admitted","clusterQueue":"cq-c22x6"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:22.12245023Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-dqfgt","name":"trainjob-test-preemption-trainjob-t7bj6-a698f","reconcileID":"f63a3adb-7083-44a7-8a35-919c399cde0a"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:22.122510848Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-dqfgt","name":"trainjob-test-preemption-trainjob-t7bj6-a698f","reconcileID":"f63a3adb-7083-44a7-8a35-919c399cde0a","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-04T04:24:22.122566907Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-dqfgt","name":"trainjob-test-preemption-trainjob-t7bj6-a698f","reconcileID":"f711376c-1d7f-4799-b90f-2b5c13d46d67"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:22.169954632Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-dqfgt","name":"test-preemption-trainjob-t7bj6","reconcileID":"b0ce41d3-0641-4171-b674-9e94aa562958","job":"test-ns-dqfgt/test-preemption-trainjob-t7bj6","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-11-04T04:24:22.496287803Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-dqfgt","name":"test-preemption-trainjob-t7bj6","reconcileID":"b0ce41d3-0641-4171-b674-9e94aa562958","error":"jobs are still active","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"debug","ts":"2026-11-04T04:24:22.496308871Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"test-preemption-trainjob-t7bj6","namespace":"test-ns-dqfgt"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"Stopped","action":"Stopped","note":"The workload is deactivated"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:22.496441252Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-dqfgt","name":"test-preemption-trainjob-t7bj6","reconcileID":"f13b605b-5c04-4284-b0eb-5ed113844107","job":"test-ns-dqfgt/test-preemption-trainjob-t7bj6","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-11-04T04:24:22.654699093Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-dqfgt","name":"test-preemption-trainjob-t7bj6","reconcileID":"f13b605b-5c04-4284-b0eb-5ed113844107","error":"jobs are still active","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:22.654827831Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-dqfgt","name":"test-preemption-trainjob-t7bj6","reconcileID":"dd601e37-92ac-4fdf-a3e4-18ab20515351","job":"test-ns-dqfgt/test-preemption-trainjob-t7bj6","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:22.760125185Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-dqfgt","name":"trainjob-test-preemption-trainjob-t7bj6-a698f","reconcileID":"a00bca52-541b-4955-9d7e-fc211c3e5ede"}
{"level":"error","ts":"2026-11-04T04:24:22.812885495Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-dqfgt","name":"test-preemption-trainjob-t7bj6","reconcileID":"dd601e37-92ac-4fdf-a3e4-18ab20515351","error":"jobs are still active","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:22.812998718Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-dqfgt","name":"test-preemption-trainjob-t7bj6","reconcileID":"8fb9ef53-8b10-47aa-82a3-7883e88b62d2","job":"test-ns-dqfgt/test-preemption-trainjob-t7bj6","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-11-04T04:24:22.97097119Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-dqfgt","name":"test-preemption-trainjob-t7bj6","reconcileID":"8fb9ef53-8b10-47aa-82a3-7883e88b62d2","error":"jobs are still active","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:22.971065995Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-dqfgt","name":"test-preemption-trainjob-t7bj6","reconcileID":"b3ba2d7d-1616-48ec-89a6-c483285d3438","job":"test-ns-dqfgt/test-preemption-trainjob-t7bj6","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.011811274Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-dqfgt","name":"lq-bl6f8","reconcileID":"abad7932-840c-416c-a44c-7a2aad812536"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.011823698Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-c22x6","reconcileID":"f0476880-d1fe-4fb3-aa6f-1cf06a0085df"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.067103645Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-bl6f8","namespace":"test-ns-dqfgt"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.067214494Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-dqfgt","name":"lq-bl6f8","reconcileID":"a803bb93-7fa1-4662-b42d-b290e916ea8f"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.0700757Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-c22x6"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.07027343Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-c22x6","reconcileID":"a6751458-8a96-4dc1-8a4b-e30af72509c3"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.077039687Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-c22x6"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.077098669Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-c22x6","reconcileID":"670ea85a-1203-4455-b70d-cbbe2a9fc17c"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.077154965Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-dqfgt","name":"trainjob-test-preemption-trainjob-t7bj6-a698f","reconcileID":"8bd45106-3861-41e7-af2b-f5b6b7836fd9"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.092496229Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:171","msg":"ResourceFlavor update event","replica-role":"leader","resourceFlavor":{"name":"rf-rt7ms"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.092549695Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-rt7ms","reconcileID":"6790e84e-ca37-43f6-a805-24fc6aaf7ed2"}
{"level":"error","ts":"2026-11-04T04:24:23.12886635Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-dqfgt","name":"test-preemption-trainjob-t7bj6","reconcileID":"b3ba2d7d-1616-48ec-89a6-c483285d3438","error":"jobs are still active","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.128946699Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-dqfgt","name":"test-preemption-trainjob-t7bj6","reconcileID":"8f454087-4ed3-4498-86f6-b4fb86c7b694","job":"test-ns-dqfgt/test-preemption-trainjob-t7bj6","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.189372494Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:376","msg":"ClusterQueue delete event","replica-role":"leader","clusterQueue":{"name":"cq-c22x6"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.189441081Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:385","msg":"Cleared resource metrics for deleted ClusterQueue.","replica-role":"leader","clusterQueue":{"name":"cq-c22x6"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.189489016Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-dqfgt","name":"lq-bl6f8","reconcileID":"25dae600-371f-4a2a-b8a5-2e400e62fbc1"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.189522938Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-dqfgt","name":"trainjob-test-preemption-trainjob-t7bj6-a698f","reconcileID":"d7dc79be-2519-438b-afbe-609897ce3fde"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.189563641Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-rt7ms","reconcileID":"cb89371f-14c7-4ceb-bd69-4058ac083b34"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.240855735Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-bl6f8","namespace":"test-ns-dqfgt"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.241109382Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-dqfgt","name":"lq-bl6f8","reconcileID":"8d8d2c32-51e4-4a5a-9e02-954828f34a27"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.288576748Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:159","msg":"ResourceFlavor delete event","replica-role":"leader","resourceFlavor":{"name":"rf-rt7ms"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.377388483Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:141","msg":"ResourceFlavor create event","replica-role":"leader","resourceFlavor":{"name":"rf-4g785"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.37745947Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-4g785","reconcileID":"4e53fab7-ba6a-4d92-b03a-6aa867807436"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.507325268Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:351","msg":"ClusterQueue create event","replica-role":"leader","clusterQueue":{"name":"cq-44w2t"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.507486758Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-44w2t","reconcileID":"6d44fe15-d486-4fa4-ba12-0d69cbffa325"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.536394416Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:240","msg":"LocalQueue create event","replica-role":"leader","localQueue":{"name":"lq-48blb","namespace":"test-ns-whn67"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.536495052Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-whn67","name":"lq-48blb","reconcileID":"444cb1b0-78f3-47a6-8a4e-a3b7edbd53c2"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.57962106Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-dqfgt","name":"test-preemption-trainjob-t7bj6","reconcileID":"5a55a40b-3f3f-4b57-bd20-5fa0a15d5c58","job":"test-ns-dqfgt/test-preemption-trainjob-t7bj6","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.579678461Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-44w2t"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.579816429Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-44w2t","reconcileID":"f3d3758e-6e68-4b59-97f4-fc27efd6237f"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.580346646Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-t7bj6-a698f","namespace":"test-ns-dqfgt"},"queue":"lq-bl6f8","status":"pending","prevStatus":"admitted","prevClusterQueue":"cq-c22x6"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.580371715Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-dqfgt","name":"trainjob-test-preemption-trainjob-t7bj6-a698f","reconcileID":"64ad8b24-1f42-4bb1-aa0f-bba3bbde36f9"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.580393577Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1381","msg":"Workload will not be queued because the workload is not active","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-t7bj6-a698f","namespace":"test-ns-dqfgt"},"queue":"lq-bl6f8","status":"pending","prevStatus":"admitted","prevClusterQueue":"cq-c22x6"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.58040926Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-dqfgt","name":"trainjob-test-preemption-trainjob-t7bj6-a698f","reconcileID":"64ad8b24-1f42-4bb1-aa0f-bba3bbde36f9","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.580457251Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-dqfgt","name":"trainjob-test-preemption-trainjob-t7bj6-a698f","reconcileID":"3cc89b10-c503-43c3-9a6d-02abb15c7389"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.590484298Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-whn67","name":"lq-48blb","reconcileID":"130d17a3-7676-4f3f-b028-4a232ba2d5c6"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.590517926Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-48blb","namespace":"test-ns-whn67"}}
{"level":"error","ts":"2026-11-04T04:24:23.629276783Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-dqfgt","name":"test-preemption-trainjob-t7bj6","reconcileID":"5a55a40b-3f3f-4b57-bd20-5fa0a15d5c58","job":"test-ns-dqfgt/test-preemption-trainjob-t7bj6","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-t7bj6\" is forbidden: unable to create new content in namespace test-ns-dqfgt because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-04T04:24:23.629332Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-dqfgt","name":"test-preemption-trainjob-t7bj6","reconcileID":"5a55a40b-3f3f-4b57-bd20-5fa0a15d5c58","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-t7bj6\" is forbidden: unable to create new content in namespace test-ns-dqfgt because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.629424063Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-dqfgt","name":"test-preemption-trainjob-t7bj6","reconcileID":"9297c880-19d5-48ce-b4d3-982fa7ee173e","job":"test-ns-dqfgt/test-preemption-trainjob-t7bj6","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-3)","ts":"2026-11-04T04:24:23.640059891Z","logger":"localqueue-reconciler","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-whn67","name":"lq-48blb","reconcileID":"130d17a3-7676-4f3f-b028-4a232ba2d5c6","error":"Operation cannot be fulfilled on localqueues.kueue.x-k8s.io \"lq-48blb\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.640119388Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-whn67","name":"lq-48blb","reconcileID":"55f6b49c-4bbd-4368-9ea1-eba4809fa9dd"}
{"level":"error","ts":"2026-11-04T04:24:23.67733004Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-dqfgt","name":"test-preemption-trainjob-t7bj6","reconcileID":"9297c880-19d5-48ce-b4d3-982fa7ee173e","job":"test-ns-dqfgt/test-preemption-trainjob-t7bj6","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-t7bj6\" is forbidden: unable to create new content in namespace test-ns-dqfgt because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-04T04:24:23.677379813Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-dqfgt","name":"test-preemption-trainjob-t7bj6","reconcileID":"9297c880-19d5-48ce-b4d3-982fa7ee173e","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-t7bj6\" is forbidden: unable to create new content in namespace test-ns-dqfgt because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.687622722Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-dqfgt","name":"test-preemption-trainjob-t7bj6","reconcileID":"18b708be-3df1-4295-928a-7d1c42a8e29a","job":"test-ns-dqfgt/test-preemption-trainjob-t7bj6","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.693329487Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-t7bj6-a698f","namespace":"test-ns-dqfgt"},"queue":"lq-bl6f8","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.693351511Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-dqfgt","name":"trainjob-test-preemption-trainjob-t7bj6-a698f","reconcileID":"79ad919b-3814-4a3f-8643-b84c7352ba73"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.69338304Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-dqfgt","name":"trainjob-test-preemption-trainjob-t7bj6-a698f","reconcileID":"79ad919b-3814-4a3f-8643-b84c7352ba73","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.693381024Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1381","msg":"Workload will not be queued because the workload is not active","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-t7bj6-a698f","namespace":"test-ns-dqfgt"},"queue":"lq-bl6f8","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.69344031Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-dqfgt","name":"trainjob-test-preemption-trainjob-t7bj6-a698f","reconcileID":"c70fb21c-9d42-4d85-af46-c27ae822fe8f"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.695505132Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-48blb","namespace":"test-ns-whn67"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.69558729Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-whn67","name":"lq-48blb","reconcileID":"5e41fb7b-2c81-45e7-9faf-bc6f7cd6cb0e"}
{"level":"error","ts":"2026-11-04T04:24:23.73578686Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-dqfgt","name":"test-preemption-trainjob-t7bj6","reconcileID":"18b708be-3df1-4295-928a-7d1c42a8e29a","job":"test-ns-dqfgt/test-preemption-trainjob-t7bj6","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-t7bj6\" is forbidden: unable to create new content in namespace test-ns-dqfgt because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-04T04:24:23.735840094Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-dqfgt","name":"test-preemption-trainjob-t7bj6","reconcileID":"18b708be-3df1-4295-928a-7d1c42a8e29a","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-t7bj6\" is forbidden: unable to create new content in namespace test-ns-dqfgt because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.735913255Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-dqfgt","name":"test-preemption-trainjob-t7bj6","reconcileID":"3ed73055-177a-437d-9dda-3a14bffd03b1","job":"test-ns-dqfgt/test-preemption-trainjob-t7bj6","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-11-04T04:24:23.784236232Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-dqfgt","name":"test-preemption-trainjob-t7bj6","reconcileID":"3ed73055-177a-437d-9dda-3a14bffd03b1","job":"test-ns-dqfgt/test-preemption-trainjob-t7bj6","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-t7bj6\" is forbidden: unable to create new content in namespace test-ns-dqfgt because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-04T04:24:23.784288762Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-dqfgt","name":"test-preemption-trainjob-t7bj6","reconcileID":"3ed73055-177a-437d-9dda-3a14bffd03b1","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-t7bj6\" is forbidden: unable to create new content in namespace test-ns-dqfgt because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.784376804Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-whn67","name":"test-runtimepatches-trainjob-w2wqc","reconcileID":"3ef9691a-b5dc-4eb8-8ecd-ee16ab28b6ba","job":"test-ns-whn67/test-runtimepatches-trainjob-w2wqc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:24.103602496Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1276","msg":"Workload create event","replica-role":"leader","workload":{"name":"trainjob-test-runtimepatches-trainjob-w2wqc-fbb90","namespace":"test-ns-whn67"},"queue":"lq-48blb","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:24.103692986Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-whn67","name":"trainjob-test-runtimepatches-trainjob-w2wqc-fbb90","reconcileID":"606cc07c-c9a8-4569-aa35-3538f8b75774"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:24.103733731Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-whn67","name":"trainjob-test-runtimepatches-trainjob-w2wqc-fbb90","reconcileID":"606cc07c-c9a8-4569-aa35-3538f8b75774","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-04T04:24:24.10374168Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-dqfgt","name":"test-preemption-trainjob-t7bj6","reconcileID":"18a27420-f037-4c97-b4e8-4e020aa50565","job":"test-ns-dqfgt/test-preemption-trainjob-t7bj6","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"debug","ts":"2026-11-04T04:24:24.103740613Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"test-runtimepatches-trainjob-w2wqc","namespace":"test-ns-whn67"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"CreatedWorkload","action":"CreatedWorkload","note":"Created Workload: test-ns-whn67/trainjob-test-runtimepatches-trainjob-w2wqc-fbb90"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:24.10383374Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-whn67","name":"trainjob-test-runtimepatches-trainjob-w2wqc-fbb90","reconcileID":"c3006a20-d223-4e93-8f75-f7399b5a4c1d"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:24.103863586Z","logger":"scheduler","caller":"scheduler/scheduler.go:308","msg":"Obtained heads","replica-role":"leader","schedulingCycle":4,"headCount":1,"waitDuration":4.931880622}
{"level":"Level(-2)","ts":"2026-11-04T04:24:24.103896978Z","logger":"scheduler","caller":"scheduler/scheduler.go:323","msg":"Snapshot taken","replica-role":"leader","schedulingCycle":4,"duration":0.000012081}
{"level":"Level(-2)","ts":"2026-11-04T04:24:24.10397942Z","logger":"scheduler","caller":"scheduler/scheduler.go:328","msg":"Nomination done","replica-role":"leader","schedulingCycle":4,"entries":1,"inadmissibleEntries":0,"duration":0.000071259}
{"level":"Level(-2)","ts":"2026-11-04T04:24:24.10400579Z","logger":"scheduler","caller":"scheduler/scheduler.go:406","msg":"Attempting to schedule workload","replica-role":"leader","schedulingCycle":4,"workload":{"name":"trainjob-test-runtimepatches-trainjob-w2wqc-fbb90","namespace":"test-ns-whn67"},"clusterQueue":{"name":"cq-44w2t"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:24.104100284Z","logger":"scheduler","caller":"scheduler/scheduler.go:1041","msg":"Workload assumed in the cache","replica-role":"leader","schedulingCycle":4,"workload":{"name":"trainjob-test-runtimepatches-trainjob-w2wqc-fbb90","namespace":"test-ns-whn67"},"clusterQueue":{"name":"cq-44w2t"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:24.10412373Z","logger":"scheduler","caller":"scheduler/scheduler.go:362","msg":"Workload processing done","replica-role":"leader","schedulingCycle":4,"duration":0.000122934}
{"level":"Level(-2)","ts":"2026-11-04T04:24:24.104138164Z","logger":"scheduler","caller":"scheduler/scheduler.go:297","msg":"Scheduling cycle complete","replica-role":"leader","schedulingCycle":4,"duration":4.932155912}
{"level":"Level(-2)","ts":"2026-11-04T04:24:24.104155677Z","logger":"scheduler","caller":"scheduler/scheduler.go:295","msg":"Scheduling cycle starts","replica-role":"leader","schedulingCycle":5}
{"level":"error","ts":"2026-11-04T04:24:24.151819708Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-dqfgt","name":"test-preemption-trainjob-t7bj6","reconcileID":"18a27420-f037-4c97-b4e8-4e020aa50565","job":"test-ns-dqfgt/test-preemption-trainjob-t7bj6","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-t7bj6\" is forbidden: unable to create new content in namespace test-ns-dqfgt because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-04T04:24:24.151878582Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-dqfgt","name":"test-preemption-trainjob-t7bj6","reconcileID":"18a27420-f037-4c97-b4e8-4e020aa50565","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-t7bj6\" is forbidden: unable to create new content in namespace test-ns-dqfgt because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:24.151957616Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-whn67","name":"test-runtimepatches-trainjob-w2wqc","reconcileID":"f54b8766-8f90-4053-97d9-becac2398a99","job":"test-ns-whn67/test-runtimepatches-trainjob-w2wqc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:24.214069877Z","logger":"scheduler","caller":"scheduler/scheduler.go:1000","msg":"Workload successfully admitted and assigned flavors","replica-role":"leader","schedulingCycle":4,"workload":{"name":"trainjob-test-runtimepatches-trainjob-w2wqc-fbb90","namespace":"test-ns-whn67"},"clusterQueue":{"name":"cq-44w2t"},"assignments":[{"name":"node","flavors":{"cpu":"rf-4g785","memory":"rf-4g785"},"resourceUsage":{"cpu":"100m","memory":"128Mi"},"count":1}]}
{"level":"debug","ts":"2026-11-04T04:24:24.214096035Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-test-runtimepatches-trainjob-w2wqc-fbb90","namespace":"test-ns-whn67"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"QuotaReserved","action":"QuotaReserved","note":"Quota reserved in ClusterQueue cq-44w2t, wait time since queued was 0s"}
{"level":"debug","ts":"2026-11-04T04:24:24.214124187Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-test-runtimepatches-trainjob-w2wqc-fbb90","namespace":"test-ns-whn67"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"Admitted","action":"Admitted","note":"Admitted by ClusterQueue cq-44w2t, wait time since reservation was 0s"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:24.214240404Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-runtimepatches-trainjob-w2wqc-fbb90","namespace":"test-ns-whn67"},"queue":"lq-48blb","status":"admitted","prevStatus":"pending","clusterQueue":"cq-44w2t"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:24.214311055Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-whn67","name":"trainjob-test-runtimepatches-trainjob-w2wqc-fbb90","reconcileID":"1a8cc0a4-ba7b-46c7-8bd4-91bbabeaa14a"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:24.214358936Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-whn67","name":"trainjob-test-runtimepatches-trainjob-w2wqc-fbb90","reconcileID":"1a8cc0a4-ba7b-46c7-8bd4-91bbabeaa14a","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-04T04:24:24.214442867Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-whn67","name":"trainjob-test-runtimepatches-trainjob-w2wqc-fbb90","reconcileID":"b7bfef71-562c-44f2-9af7-3beb2f05059f"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:24.310054399Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-whn67","name":"test-runtimepatches-trainjob-w2wqc","reconcileID":"141ab7a1-00c0-444a-a3fd-34390cf8454b","job":"test-ns-whn67/test-runtimepatches-trainjob-w2wqc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:24.468743372Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-whn67","name":"test-runtimepatches-trainjob-w2wqc","reconcileID":"141ab7a1-00c0-444a-a3fd-34390cf8454b","job":"test-ns-whn67/test-runtimepatches-trainjob-w2wqc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-3)","ts":"2026-11-04T04:24:24.577685771Z","logger":"trainjob","caller":"jobframework/reconciler.go:594","msg":"Unsuspending job","replica-role":"leader","namespace":"test-ns-whn67","name":"test-runtimepatches-trainjob-w2wqc","reconcileID":"141ab7a1-00c0-444a-a3fd-34390cf8454b","job":"test-ns-whn67/test-runtimepatches-trainjob-w2wqc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-runtimepatches-trainjob-w2wqc\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-3)","ts":"2026-11-04T04:24:24.577752476Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-whn67","name":"test-runtimepatches-trainjob-w2wqc","reconcileID":"141ab7a1-00c0-444a-a3fd-34390cf8454b","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-runtimepatches-trainjob-w2wqc\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:24.577837878Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-dqfgt","name":"test-preemption-trainjob-t7bj6","reconcileID":"359d96d2-1ae8-44f4-906e-1eafb1191ee7","job":"test-ns-dqfgt/test-preemption-trainjob-t7bj6","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:24.580797479Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-dqfgt","name":"lq-bl6f8","reconcileID":"75d02839-bbed-4708-8abb-f46acc5c1955"}
{"level":"error","ts":"2026-11-04T04:24:24.626012328Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-dqfgt","name":"test-preemption-trainjob-t7bj6","reconcileID":"359d96d2-1ae8-44f4-906e-1eafb1191ee7","job":"test-ns-dqfgt/test-preemption-trainjob-t7bj6","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-t7bj6\" is forbidden: unable to create new content in namespace test-ns-dqfgt because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-04T04:24:24.626065634Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-dqfgt","name":"test-preemption-trainjob-t7bj6","reconcileID":"359d96d2-1ae8-44f4-906e-1eafb1191ee7","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-t7bj6\" is forbidden: unable to create new content in namespace test-ns-dqfgt because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:24.626135306Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-whn67","name":"test-runtimepatches-trainjob-w2wqc","reconcileID":"1e4e176b-a18d-40b4-b272-22c9ad85a1c4","job":"test-ns-whn67/test-runtimepatches-trainjob-w2wqc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:24.784709277Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-whn67","name":"test-runtimepatches-trainjob-w2wqc","reconcileID":"1e4e176b-a18d-40b4-b272-22c9ad85a1c4","job":"test-ns-whn67/test-runtimepatches-trainjob-w2wqc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:24.809212669Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-44w2t"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:24.809305961Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-44w2t","reconcileID":"e525c7d7-87cb-459b-a572-d58b11747b5d"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:24.809337919Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-whn67","name":"trainjob-test-runtimepatches-trainjob-w2wqc-fbb90","reconcileID":"4f93f355-e786-493e-bee6-5af4f472cfac"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:24.825009708Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:171","msg":"ResourceFlavor update event","replica-role":"leader","resourceFlavor":{"name":"rf-4g785"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:24.825066606Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-4g785","reconcileID":"5a3e12b2-6d4f-4874-a880-7ba13c2fb8b5"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:24.864610656Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-44w2t"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:24.864683496Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-44w2t","reconcileID":"272ee638-3cd0-41b7-8cfb-039b74fb5487"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:24.864695535Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-whn67","name":"lq-48blb","reconcileID":"2d98dd5d-45ee-4439-8765-485ee58050f4"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:24.864725013Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-whn67","name":"trainjob-test-runtimepatches-trainjob-w2wqc-fbb90","reconcileID":"476384b8-f03e-4112-81ad-0a49d407f83c"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:24.919334429Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-48blb","namespace":"test-ns-whn67"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:24.919414318Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-whn67","name":"lq-48blb","reconcileID":"89aee08f-556e-40a3-b2d9-0ebdbc4a7d49"}