{"level":"Level(-2)","ts":"2026-11-02T21:56:40.158685235Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-sfndj","name":"progression-invalid","reconcileID":"3e28cccb-9342-44ab-bd09-7d9e0a27cae4","job":"rhai-e2e-progression-sfndj/progression-invalid","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-sfndj"}
{"level":"Level(-2)","ts":"2026-11-02T21:56:41.167006063Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-sfndj","name":"progression-invalid","reconcileID":"27e31b8f-0c33-453b-b23c-fbad11621642","job":"rhai-e2e-progression-sfndj/progression-invalid","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-sfndj"}
{"level":"Level(-2)","ts":"2026-11-02T21:56:41.319878598Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-sfndj","name":"progression-invalid","reconcileID":"9e14f243-5667-4df1-9c96-b7006a1e2f99","job":"rhai-e2e-progression-sfndj/progression-invalid","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-sfndj"}
{"level":"Level(-2)","ts":"2026-11-02T21:56:41.758451659Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-sfndj","name":"progression-invalid","reconcileID":"3636b0cf-000b-4e92-be49-8bf8855748a9","job":"rhai-e2e-progression-sfndj/progression-invalid","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-sfndj"}
{"level":"Level(-2)","ts":"2026-11-02T21:58:09.562334307Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-sfndj","name":"progression-custom-config","reconcileID":"60812d72-bfea-466a-9029-e02d96ad6943","job":"rhai-e2e-progression-sfndj/progression-custom-config","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-sfndj"}
{"level":"Level(-2)","ts":"2026-11-02T21:58:09.619834749Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-sfndj","name":"progression-custom-config","reconcileID":"b5bde700-58c0-4219-a446-0ec4b6b2105c","job":"rhai-e2e-progression-sfndj/progression-custom-config","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-sfndj"}
{"level":"Level(-2)","ts":"2026-11-02T21:58:12.085491681Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-sfndj","name":"progression-job-failure","reconcileID":"96cec801-2b58-4a62-8847-d4dfb8cf93e7","job":"rhai-e2e-progression-sfndj/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-sfndj"}
{"level":"Level(-2)","ts":"2026-11-02T21:58:12.191505655Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-sfndj","name":"progression-job-failure","reconcileID":"8e0b5f2f-b98c-4e4b-9d3c-e6f8a5731fea","job":"rhai-e2e-progression-sfndj/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-sfndj"}
{"level":"Level(-2)","ts":"2026-11-02T21:58:12.466218973Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-sfndj","name":"progression-job-failure","reconcileID":"79b4e30f-6508-4070-8cea-a5aa39a0793b","job":"rhai-e2e-progression-sfndj/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-sfndj"}
{"level":"Level(-2)","ts":"2026-11-02T21:58:12.680450718Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-sfndj","name":"progression-job-failure","reconcileID":"0cb6aa14-5c8a-4bfc-bec1-62418a1c3a96","job":"rhai-e2e-progression-sfndj/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-sfndj"}
{"level":"Level(-2)","ts":"2026-11-02T21:58:19.136596833Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-sfndj","name":"progression-job-failure","reconcileID":"c78beb00-36ec-42b9-92d2-627ba688de5c","job":"rhai-e2e-progression-sfndj/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-sfndj"}
{"level":"Level(-2)","ts":"2026-11-02T21:58:19.245298713Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-sfndj","name":"progression-job-failure","reconcileID":"a00dc0e0-2320-4536-b842-58282817ae27","job":"rhai-e2e-progression-sfndj/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-sfndj"}
{"level":"Level(-2)","ts":"2026-11-02T21:58:20.143341234Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-sfndj","name":"progression-job-failure","reconcileID":"7519b20c-36fb-4ea7-b7f2-b34d6fef1457","job":"rhai-e2e-progression-sfndj/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-sfndj"}
{"level":"Level(-2)","ts":"2026-11-02T21:58:20.249333099Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-sfndj","name":"progression-job-failure","reconcileID":"980f9f8f-de33-4c90-918a-7b7926290874","job":"rhai-e2e-progression-sfndj/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-sfndj"}
{"level":"Level(-2)","ts":"2026-11-02T21:58:20.255087974Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-sfndj","name":"progression-job-failure","reconcileID":"1407d4e1-c893-4d85-a56e-b00ab1322766","job":"rhai-e2e-progression-sfndj/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-sfndj"}
{"level":"Level(-2)","ts":"2026-11-02T21:58:20.767549058Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-sfndj","name":"progression-job-failure","reconcileID":"c16adf87-e462-4496-93be-a3e6b10d9173","job":"rhai-e2e-progression-sfndj/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-sfndj"}
{"level":"Level(-2)","ts":"2026-11-02T21:58:20.929070598Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-sfndj","name":"progression-job-failure","reconcileID":"25ba46ae-5530-42a2-a3b9-95f1c447cb0e","job":"rhai-e2e-progression-sfndj/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-sfndj"}
{"level":"Level(-2)","ts":"2026-11-02T21:58:21.14235545Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-sfndj","name":"progression-job-failure","reconcileID":"3568e319-6355-42ca-a1eb-53ec927e16a3","job":"rhai-e2e-progression-sfndj/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-sfndj"}
{"level":"Level(-2)","ts":"2026-11-02T21:58:21.685900835Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-sfndj","name":"progression-no-metrics","reconcileID":"f606a577-e67a-405d-9d1d-34af6fc25045","job":"rhai-e2e-progression-sfndj/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-sfndj"}
{"level":"Level(-2)","ts":"2026-11-02T21:58:22.080757394Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-sfndj","name":"progression-no-metrics","reconcileID":"51d22553-0f78-457e-955a-00e422becd88","job":"rhai-e2e-progression-sfndj/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-sfndj"}
{"level":"Level(-2)","ts":"2026-11-02T21:58:22.300940492Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-sfndj","name":"progression-no-metrics","reconcileID":"cc73c1ad-851a-40b8-bad5-3fce28e58038","job":"rhai-e2e-progression-sfndj/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-sfndj"}
{"level":"Level(-2)","ts":"2026-11-02T21:58:22.598062709Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-sfndj","name":"progression-no-metrics","reconcileID":"2e9a4340-9405-407d-94bc-00e62f294f9e","job":"rhai-e2e-progression-sfndj/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-sfndj"}
{"level":"Level(-2)","ts":"2026-11-02T21:58:22.710091109Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-sfndj","name":"progression-no-metrics","reconcileID":"7ae5113a-dac5-4934-978d-de94994b1cb3","job":"rhai-e2e-progression-sfndj/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-sfndj"}
{"level":"Level(-2)","ts":"2026-11-02T21:58:23.33333999Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-sfndj","name":"progression-job-failure","reconcileID":"91064cea-d85c-49ae-a774-2f0991d99505","job":"rhai-e2e-progression-sfndj/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-sfndj"}
{"level":"Level(-2)","ts":"2026-11-02T21:58:24.621005974Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-sfndj","name":"progression-no-metrics","reconcileID":"3d0f29dd-fc0c-4e55-8a66-08bf9e5bae51","job":"rhai-e2e-progression-sfndj/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-sfndj"}
{"level":"Level(-2)","ts":"2026-11-02T21:58:24.729753439Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-sfndj","name":"progression-no-metrics","reconcileID":"6980faf0-3e1c-423e-9a2a-cb59bc93590e","job":"rhai-e2e-progression-sfndj/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-sfndj"}
{"level":"Level(-2)","ts":"2026-11-02T21:58:29.646093241Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-sfndj","name":"progression-no-metrics","reconcileID":"0e79aabc-7747-48ca-96b8-295602a2ed95","job":"rhai-e2e-progression-sfndj/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-sfndj"}
{"level":"Level(-2)","ts":"2026-11-02T21:58:29.75585605Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-sfndj","name":"progression-no-metrics","reconcileID":"5e2ea9c8-679b-4000-b753-711376784317","job":"rhai-e2e-progression-sfndj/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-sfndj"}
{"level":"Level(-2)","ts":"2026-11-02T21:58:30.653574549Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-sfndj","name":"progression-no-metrics","reconcileID":"7aca432d-5342-4c28-93d4-83f0162fb728","job":"rhai-e2e-progression-sfndj/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-sfndj"}
{"level":"Level(-2)","ts":"2026-11-02T21:58:30.748032085Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-sfndj","name":"progression-custom-config","reconcileID":"a65f23a3-ec03-47b4-9cfe-5187eeec2707","job":"rhai-e2e-progression-sfndj/progression-custom-config","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-sfndj"}
{"level":"Level(-2)","ts":"2026-11-02T21:58:30.780224003Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-sfndj","name":"progression-no-metrics","reconcileID":"89ad3bd0-fcb3-4fd6-b9d1-fc036b7fa126","job":"rhai-e2e-progression-sfndj/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-sfndj"}
{"level":"Level(-2)","ts":"2026-11-02T21:58:30.80561607Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-sfndj","name":"progression-no-metrics","reconcileID":"c3404264-df6b-4c87-bae2-644714c557b8","job":"rhai-e2e-progression-sfndj/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-sfndj"}
{"level":"Level(-2)","ts":"2026-11-02T21:58:31.246092432Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-sfndj","name":"progression-custom-config","reconcileID":"cf7b366d-ee9d-4d27-a7b6-d58ad242baed","job":"rhai-e2e-progression-sfndj/progression-custom-config","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-sfndj"}
{"level":"Level(-2)","ts":"2026-11-02T21:58:31.666859025Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-sfndj","name":"progression-custom-config","reconcileID":"10a4ee9d-6c96-4f6f-b0d9-ac3e71cb7cd2","job":"rhai-e2e-progression-sfndj/progression-custom-config","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-sfndj"}
{"level":"Level(-2)","ts":"2026-11-02T21:58:31.713335706Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-sfndj","name":"progression-no-metrics","reconcileID":"7375ef55-e54c-46eb-b5a4-d099a59e09d6","job":"rhai-e2e-progression-sfndj/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-sfndj"}
{"level":"Level(-2)","ts":"2026-11-02T21:58:31.765565533Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-sfndj","name":"progression-custom-config","reconcileID":"a9cc9fd1-33c8-42e9-ad8d-6a8ad0cccd4a","job":"rhai-e2e-progression-sfndj/progression-custom-config","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-sfndj"}
{"level":"Level(-2)","ts":"2026-11-02T21:58:31.873673933Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-sfndj","name":"progression-no-metrics","reconcileID":"f9084a64-f01a-4e0a-9544-ceeefb9dc3e0","job":"rhai-e2e-progression-sfndj/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-sfndj"}
{"level":"Level(-2)","ts":"2026-11-02T21:58:31.930296659Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-sfndj","name":"progression-custom-config","reconcileID":"833b7a27-dd81-45eb-989a-1686ff78109f","job":"rhai-e2e-progression-sfndj/progression-custom-config","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-sfndj"}
{"level":"Level(-2)","ts":"2026-11-02T21:59:36.30763146Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-tv7d9","name":"test-trainjob-jwrtw","reconcileID":"6aabecd3-de0c-4b94-b155-f8db60433f04","job":"test-ns-tv7d9/test-trainjob-jwrtw","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-tv7d9"}
{"level":"Level(-2)","ts":"2026-11-02T21:59:36.488413961Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-tv7d9","name":"test-trainjob-jwrtw","reconcileID":"0d9c77df-044a-4772-a516-cb07db238fae","job":"test-ns-tv7d9/test-trainjob-jwrtw","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-tv7d9"}
{"level":"Level(-2)","ts":"2026-11-02T21:59:36.708869954Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-tv7d9","name":"test-trainjob-jwrtw","reconcileID":"67c79124-8494-4db0-bcf3-b3c5423adafb","job":"test-ns-tv7d9/test-trainjob-jwrtw","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-tv7d9"}
{"level":"Level(-2)","ts":"2026-11-02T21:59:36.814403075Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-tv7d9","name":"test-trainjob-jwrtw","reconcileID":"3c9da466-3790-4e66-93fe-7d5deef45424","job":"test-ns-tv7d9/test-trainjob-jwrtw","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-tv7d9"}
{"level":"Level(-2)","ts":"2026-11-02T21:59:36.819936271Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-tv7d9","name":"test-trainjob-jwrtw","reconcileID":"f6dd9549-b30e-4ed9-a5e0-17922444993f","job":"test-ns-tv7d9/test-trainjob-jwrtw","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-tv7d9"}
{"level":"Level(-2)","ts":"2026-11-02T21:59:36.922312762Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-tv7d9","name":"test-trainjob-jwrtw","reconcileID":"7db91056-56c5-46c3-8e1d-aaa0637b71d0","job":"test-ns-tv7d9/test-trainjob-jwrtw","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-tv7d9"}
{"level":"Level(-2)","ts":"2026-11-02T21:59:39.435621732Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-tv7d9","name":"test-trainjob-jwrtw","reconcileID":"bdacb08c-f4c6-4066-8a24-6575970cc6fd","job":"test-ns-tv7d9/test-trainjob-jwrtw","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-tv7d9"}
{"level":"Level(-2)","ts":"2026-11-02T21:59:39.541478003Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-tv7d9","name":"test-trainjob-jwrtw","reconcileID":"c8fa4dc7-46c4-4d40-9f86-211c628f229e","job":"test-ns-tv7d9/test-trainjob-jwrtw","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-tv7d9"}
{"level":"Level(-2)","ts":"2026-11-02T21:59:42.479972969Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-tv7d9","name":"test-trainjob-jwrtw","reconcileID":"3fd75490-1b46-49da-b549-2e996186acff","job":"test-ns-tv7d9/test-trainjob-jwrtw","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-tv7d9"}
{"level":"Level(-2)","ts":"2026-11-02T21:59:42.551098661Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-tv7d9","name":"test-trainjob-jwrtw","reconcileID":"11b285aa-0cbb-4ee4-a9a4-9b57ddff6d86","job":"test-ns-tv7d9/test-trainjob-jwrtw","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-tv7d9"}
{"level":"Level(-2)","ts":"2026-11-02T21:59:43.45258275Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-tv7d9","name":"test-trainjob-jwrtw","reconcileID":"fe75f213-602f-438e-ac95-272bdd39e5e2","job":"test-ns-tv7d9/test-trainjob-jwrtw","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-tv7d9"}
{"level":"Level(-2)","ts":"2026-11-02T21:59:43.511761372Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-tv7d9","name":"test-trainjob-jwrtw","reconcileID":"c20956e4-f107-4361-a295-e0515c84ec11","job":"test-ns-tv7d9/test-trainjob-jwrtw","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-tv7d9"}
{"level":"Level(-2)","ts":"2026-11-02T21:59:43.558559699Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-tv7d9","name":"test-trainjob-jwrtw","reconcileID":"b471e07f-9316-41eb-a7a3-1f5349859545","job":"test-ns-tv7d9/test-trainjob-jwrtw","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-tv7d9"}
{"level":"Level(-2)","ts":"2026-11-02T21:59:43.666654634Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-tv7d9","name":"test-trainjob-jwrtw","reconcileID":"dbe5722f-1a9c-4504-b32d-9b1e06d1aa1b","job":"test-ns-tv7d9/test-trainjob-jwrtw","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-tv7d9"}
{"level":"Level(-2)","ts":"2026-11-02T21:59:44.529387187Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-6mlpt","name":"test-trainjob-xmcl7","reconcileID":"2a33dc9b-06b0-4147-bcbd-e6adb418ace4","job":"test-ns-6mlpt/test-trainjob-xmcl7","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-6mlpt"}
{"level":"Level(-2)","ts":"2026-11-02T21:59:44.70417783Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-6mlpt","name":"test-trainjob-xmcl7","reconcileID":"15031e9a-769a-4107-b428-e213dc67b68b","job":"test-ns-6mlpt/test-trainjob-xmcl7","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-6mlpt"}
{"level":"Level(-2)","ts":"2026-11-02T21:59:44.927382155Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-6mlpt","name":"test-trainjob-xmcl7","reconcileID":"a7773a2a-6d59-4fff-aa89-d4b6745e62a8","job":"test-ns-6mlpt/test-trainjob-xmcl7","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-6mlpt"}
{"level":"Level(-2)","ts":"2026-11-02T21:59:45.033126461Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-6mlpt","name":"test-trainjob-xmcl7","reconcileID":"6156bd25-16c7-435e-ba6c-7cf213b20cc5","job":"test-ns-6mlpt/test-trainjob-xmcl7","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-6mlpt"}
{"level":"Level(-2)","ts":"2026-11-02T21:59:45.038266714Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-6mlpt","name":"test-trainjob-xmcl7","reconcileID":"53acaa72-2956-498f-8727-cbde3a2ddf36","job":"test-ns-6mlpt/test-trainjob-xmcl7","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-6mlpt"}
{"level":"Level(-2)","ts":"2026-11-02T21:59:45.138776702Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-6mlpt","name":"test-trainjob-xmcl7","reconcileID":"ae311ed4-7c82-4130-a5ef-fbdc515b68bf","job":"test-ns-6mlpt/test-trainjob-xmcl7","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-6mlpt"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:21.293869925Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-6mlpt","name":"test-trainjob-xmcl7","reconcileID":"d00c6c27-da7c-48cb-bb06-b1b4cb5ac8eb","job":"test-ns-6mlpt/test-trainjob-xmcl7","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-6mlpt"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:21.40068067Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-6mlpt","name":"test-trainjob-xmcl7","reconcileID":"a48669c6-4f48-4611-a123-6d428e0b3d23","job":"test-ns-6mlpt/test-trainjob-xmcl7","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-6mlpt"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:24.252769488Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-6mlpt","name":"test-trainjob-xmcl7","reconcileID":"d756e7a6-ef8e-47ba-93b2-8de405b50df5","job":"test-ns-6mlpt/test-trainjob-xmcl7","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-6mlpt"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:24.360733468Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-6mlpt","name":"test-trainjob-xmcl7","reconcileID":"0da60d14-95a3-458f-99e6-322f8ee670c7","job":"test-ns-6mlpt/test-trainjob-xmcl7","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-6mlpt"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:25.258458117Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-6mlpt","name":"test-trainjob-xmcl7","reconcileID":"34323260-ffb1-4064-83fb-5a641f5ec710","job":"test-ns-6mlpt/test-trainjob-xmcl7","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-6mlpt"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:25.366627032Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-6mlpt","name":"test-trainjob-xmcl7","reconcileID":"129b2f1b-a0c3-4560-9d14-72a60c21df8b","job":"test-ns-6mlpt/test-trainjob-xmcl7","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-6mlpt"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:25.372171922Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-6mlpt","name":"test-trainjob-xmcl7","reconcileID":"cd9b2e63-29f1-44f7-b92c-49ba5a0f5f36","job":"test-ns-6mlpt/test-trainjob-xmcl7","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-6mlpt"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:25.474639318Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-6mlpt","name":"test-trainjob-xmcl7","reconcileID":"bbc7f463-8ab1-4801-abcd-2ade9f3571b5","job":"test-ns-6mlpt/test-trainjob-xmcl7","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-6mlpt"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:26.70206468Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-9bx8d","name":"test-trainjob-6l8tl","reconcileID":"1444e776-99c2-4158-8687-374ae21511b5","job":"test-ns-9bx8d/test-trainjob-6l8tl","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-9bx8d"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:27.084394065Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-9bx8d","name":"test-trainjob-6l8tl","reconcileID":"07ae4ab2-fe71-4bea-af8c-01c4e52c365f","job":"test-ns-9bx8d/test-trainjob-6l8tl","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-9bx8d"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:27.252916365Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-9bx8d","name":"test-trainjob-6l8tl","reconcileID":"a7736ff2-4800-4b61-85b7-a610243b671b","job":"test-ns-9bx8d/test-trainjob-6l8tl","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-9bx8d"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:27.358782562Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-9bx8d","name":"test-trainjob-6l8tl","reconcileID":"25d73940-ed41-489c-b2b1-99275cd01f76","job":"test-ns-9bx8d/test-trainjob-6l8tl","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-9bx8d"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:27.642696736Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-9bx8d","name":"test-trainjob-6l8tl","reconcileID":"2b57f570-8575-47f7-8ca6-576d633715c4","job":"test-ns-9bx8d/test-trainjob-6l8tl","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-9bx8d"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:27.750936827Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-9bx8d","name":"test-trainjob-6l8tl","reconcileID":"5878acea-427c-4995-bbe7-fbbcda74750c","job":"test-ns-9bx8d/test-trainjob-6l8tl","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-9bx8d"}
{"level":"Level(-2)","ts":"2026-11-02T22:09:19.041931044Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-9bx8d","name":"test-trainjob-6l8tl","reconcileID":"73100dad-803e-4fae-9930-6f901119e05a","job":"test-ns-9bx8d/test-trainjob-6l8tl","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-9bx8d"}
{"level":"Level(-2)","ts":"2026-11-02T22:09:19.149910342Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-9bx8d","name":"test-trainjob-6l8tl","reconcileID":"9f627b22-9c68-42b6-b075-75cd08793982","job":"test-ns-9bx8d/test-trainjob-6l8tl","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-9bx8d"}
{"level":"Level(-2)","ts":"2026-11-02T22:09:22.051549272Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-9bx8d","name":"test-trainjob-6l8tl","reconcileID":"7bdbf051-5205-482d-b84a-b118252b793e","job":"test-ns-9bx8d/test-trainjob-6l8tl","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-9bx8d"}
{"level":"Level(-2)","ts":"2026-11-02T22:09:22.157610408Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-9bx8d","name":"test-trainjob-6l8tl","reconcileID":"d8097d13-bd1a-4499-aafc-29c0d78899d8","job":"test-ns-9bx8d/test-trainjob-6l8tl","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-9bx8d"}
{"level":"Level(-2)","ts":"2026-11-02T22:09:23.059071903Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-9bx8d","name":"test-trainjob-6l8tl","reconcileID":"e4aea716-d842-4a22-a62e-b9724025771b","job":"test-ns-9bx8d/test-trainjob-6l8tl","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-9bx8d"}
{"level":"Level(-2)","ts":"2026-11-02T22:09:23.117860558Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-9bx8d","name":"test-trainjob-6l8tl","reconcileID":"282ed7dd-0702-45d9-8503-7884ed73ccad","job":"test-ns-9bx8d/test-trainjob-6l8tl","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-9bx8d"}
{"level":"Level(-2)","ts":"2026-11-02T22:09:23.165736063Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-9bx8d","name":"test-trainjob-6l8tl","reconcileID":"64152a46-da7e-40fb-a182-7fdde22d7327","job":"test-ns-9bx8d/test-trainjob-6l8tl","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-9bx8d"}
{"level":"Level(-2)","ts":"2026-11-02T22:09:23.272224292Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-9bx8d","name":"test-trainjob-6l8tl","reconcileID":"770d2128-d7d2-4a29-a6f5-9447d4a3b7c1","job":"test-ns-9bx8d/test-trainjob-6l8tl","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-9bx8d"}
{"level":"Level(-2)","ts":"2026-11-02T22:09:24.645272789Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-vhmn6","name":"test-trainjob-5pj7c","reconcileID":"2c7f21b8-b824-4706-980e-9e5724b748dc","job":"test-ns-vhmn6/test-trainjob-5pj7c","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-vhmn6"}
{"level":"Level(-2)","ts":"2026-11-02T22:09:25.035121021Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-vhmn6","name":"test-trainjob-5pj7c","reconcileID":"024b7f86-b94d-4e0a-8002-98fd1dcaeec4","job":"test-ns-vhmn6/test-trainjob-5pj7c","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-vhmn6"}
{"level":"Level(-2)","ts":"2026-11-02T22:09:25.208642702Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-vhmn6","name":"test-trainjob-5pj7c","reconcileID":"8150b614-d79c-4f92-92d5-a2b6c8055ce0","job":"test-ns-vhmn6/test-trainjob-5pj7c","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-vhmn6"}
{"level":"Level(-2)","ts":"2026-11-02T22:09:25.318418875Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-vhmn6","name":"test-trainjob-5pj7c","reconcileID":"ee11c2e2-248c-4d90-bc1e-c629179f7624","job":"test-ns-vhmn6/test-trainjob-5pj7c","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-vhmn6"}
{"level":"Level(-2)","ts":"2026-11-02T22:09:25.552568704Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-vhmn6","name":"test-trainjob-5pj7c","reconcileID":"f0adc735-07f9-412d-a206-ce29cf045cca","job":"test-ns-vhmn6/test-trainjob-5pj7c","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-vhmn6"}
{"level":"Level(-2)","ts":"2026-11-02T22:09:25.662472154Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-vhmn6","name":"test-trainjob-5pj7c","reconcileID":"9684a4fa-7a00-4a56-b01d-4e314ef819ff","job":"test-ns-vhmn6/test-trainjob-5pj7c","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-vhmn6"}
{"level":"Level(-2)","ts":"2026-11-02T22:09:29.623671519Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-9bx8d","name":"test-trainjob-6l8tl","reconcileID":"612997e1-42ae-4408-9e1b-6f5095b2b014","job":"test-ns-9bx8d/test-trainjob-6l8tl","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-9bx8d"}
{"level":"Level(-2)","ts":"2026-11-02T22:09:29.909710231Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-6mlpt","name":"test-trainjob-xmcl7","reconcileID":"c63e4bc7-8a9b-4707-8352-8916af8dff58","job":"test-ns-6mlpt/test-trainjob-xmcl7","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-6mlpt"}
{"level":"Level(-2)","ts":"2026-11-02T22:09:30.239212457Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-tv7d9","name":"test-trainjob-jwrtw","reconcileID":"d0ec1115-ae9e-480c-a4a8-807ef37a1f43","job":"test-ns-tv7d9/test-trainjob-jwrtw","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-tv7d9"}
{"level":"Level(-2)","ts":"2026-11-02T22:09:42.499626886Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-vhmn6","name":"test-trainjob-5pj7c","reconcileID":"effbbf09-9e56-4e9d-916d-00c0a7b69461","job":"test-ns-vhmn6/test-trainjob-5pj7c","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-vhmn6"}
{"level":"Level(-2)","ts":"2026-11-02T22:09:42.609979054Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-vhmn6","name":"test-trainjob-5pj7c","reconcileID":"6e220800-0db7-4b6e-a0a8-582081cdc1db","job":"test-ns-vhmn6/test-trainjob-5pj7c","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-vhmn6"}
{"level":"Level(-2)","ts":"2026-11-02T22:09:47.525243043Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-vhmn6","name":"test-trainjob-5pj7c","reconcileID":"dc4fdbe1-8268-459a-9657-a68e684bd612","job":"test-ns-vhmn6/test-trainjob-5pj7c","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-vhmn6"}
{"level":"Level(-2)","ts":"2026-11-02T22:09:47.636544272Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-vhmn6","name":"test-trainjob-5pj7c","reconcileID":"c2adb0ab-e094-4cf2-84dd-6ebc2bcd8279","job":"test-ns-vhmn6/test-trainjob-5pj7c","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-vhmn6"}
{"level":"Level(-2)","ts":"2026-11-02T22:09:48.530502771Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-vhmn6","name":"test-trainjob-5pj7c","reconcileID":"b313f336-f3b3-4089-9c33-dc8711471b74","job":"test-ns-vhmn6/test-trainjob-5pj7c","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-vhmn6"}
{"level":"Level(-2)","ts":"2026-11-02T22:09:48.642049563Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-vhmn6","name":"test-trainjob-5pj7c","reconcileID":"cbd95c93-b5a7-41b7-87a8-6bd879d85c76","job":"test-ns-vhmn6/test-trainjob-5pj7c","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-vhmn6"}
{"level":"Level(-2)","ts":"2026-11-02T22:09:48.693143225Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-vhmn6","name":"test-trainjob-5pj7c","reconcileID":"74b9b1a4-2b04-42b4-b2a1-0e722998ca09","job":"test-ns-vhmn6/test-trainjob-5pj7c","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-vhmn6"}
{"level":"Level(-2)","ts":"2026-11-02T22:09:48.751766216Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-vhmn6","name":"test-trainjob-5pj7c","reconcileID":"8f0dc3f1-4cc0-43d0-86c1-df3e73f59ffa","job":"test-ns-vhmn6/test-trainjob-5pj7c","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-vhmn6"}
{"level":"Level(-2)","ts":"2026-11-02T22:09:48.821135093Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-vhmn6","name":"test-trainjob-5pj7c","reconcileID":"942ed7f2-960b-4afe-88d4-8a78d0a7d395","job":"test-ns-vhmn6/test-trainjob-5pj7c","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-vhmn6"}
{"level":"Level(-2)","ts":"2026-11-02T22:09:48.966036095Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-vhmn6","name":"test-trainjob-5pj7c","reconcileID":"120201db-925b-4332-93cb-c710a546044c","job":"test-ns-vhmn6/test-trainjob-5pj7c","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-vhmn6"}
{"level":"Level(-2)","ts":"2026-11-02T22:10:32.741728757Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-vhmn6","name":"test-trainjob-5pj7c","reconcileID":"21533d75-99d2-4386-8b80-e050263bd781","job":"test-ns-vhmn6/test-trainjob-5pj7c","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-vhmn6"}
{"level":"Level(-2)","ts":"2026-11-02T22:10:32.851550873Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-vhmn6","name":"test-trainjob-5pj7c","reconcileID":"139010f5-19d3-4059-a26a-103052afb3a7","job":"test-ns-vhmn6/test-trainjob-5pj7c","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-vhmn6"}
{"level":"Level(-2)","ts":"2026-11-02T22:10:43.791167Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-vhmn6","name":"test-trainjob-5pj7c","reconcileID":"e61bd761-5389-4cb7-aef8-b7c9a6ab13aa","job":"test-ns-vhmn6/test-trainjob-5pj7c","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-vhmn6"}
{"level":"Level(-2)","ts":"2026-11-02T22:10:43.903376107Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-vhmn6","name":"test-trainjob-5pj7c","reconcileID":"f599772d-6091-4a5b-85b6-90653b1c54ae","job":"test-ns-vhmn6/test-trainjob-5pj7c","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-vhmn6"}
{"level":"Level(-2)","ts":"2026-11-02T22:10:44.797941887Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-vhmn6","name":"test-trainjob-5pj7c","reconcileID":"a94979b4-910f-4473-9b8d-5aa1f547ae01","job":"test-ns-vhmn6/test-trainjob-5pj7c","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-vhmn6"}
{"level":"Level(-2)","ts":"2026-11-02T22:10:44.907545432Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-vhmn6","name":"test-trainjob-5pj7c","reconcileID":"7dc28003-dad9-41f5-b2f9-e3a3dae8bc3b","job":"test-ns-vhmn6/test-trainjob-5pj7c","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-vhmn6"}
{"level":"Level(-2)","ts":"2026-11-02T22:10:45.015059264Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-vhmn6","name":"test-trainjob-5pj7c","reconcileID":"f3a90148-17d2-4fc2-bcc1-98ca8a4453c6","job":"test-ns-vhmn6/test-trainjob-5pj7c","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-vhmn6"}
{"level":"Level(-2)","ts":"2026-11-02T22:10:45.124720577Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-vhmn6","name":"test-trainjob-5pj7c","reconcileID":"353f19db-fa2d-456d-873f-a8b6a0f82458","job":"test-ns-vhmn6/test-trainjob-5pj7c","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-vhmn6"}
{"level":"Level(-2)","ts":"2026-11-02T22:10:45.130477658Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-vhmn6","name":"test-trainjob-5pj7c","reconcileID":"dae496d3-9bbb-479d-81ad-b8bb4e050b56","job":"test-ns-vhmn6/test-trainjob-5pj7c","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-vhmn6"}
{"level":"Level(-2)","ts":"2026-11-02T22:10:45.23471998Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-vhmn6","name":"test-trainjob-5pj7c","reconcileID":"34d06931-bf70-47a3-9028-592482ceb44f","job":"test-ns-vhmn6/test-trainjob-5pj7c","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-vhmn6"}
{"level":"Level(-2)","ts":"2026-11-02T22:10:58.172368835Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-vhmn6","name":"test-trainjob-5pj7c","reconcileID":"4a5974c5-c356-4369-abc1-9c815f5e5922","job":"test-ns-vhmn6/test-trainjob-5pj7c","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-vhmn6"}
{"level":"Level(-2)","ts":"2026-11-02T22:10:58.282934328Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-vhmn6","name":"test-trainjob-5pj7c","reconcileID":"7d8bde18-3983-4b6f-aa36-2b0bd444c099","job":"test-ns-vhmn6/test-trainjob-5pj7c","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-vhmn6"}
{"level":"Level(-2)","ts":"2026-11-02T22:10:59.17980081Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-vhmn6","name":"test-trainjob-5pj7c","reconcileID":"54ff3084-d43b-42f7-8c5f-af9ba9e7f2f1","job":"test-ns-vhmn6/test-trainjob-5pj7c","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-vhmn6"}
{"level":"Level(-2)","ts":"2026-11-02T22:10:59.289331971Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-vhmn6","name":"test-trainjob-5pj7c","reconcileID":"62e33e35-3c11-460b-aa82-81dd260513bb","job":"test-ns-vhmn6/test-trainjob-5pj7c","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-vhmn6"}
{"level":"Level(-2)","ts":"2026-11-02T22:11:00.188031355Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-vhmn6","name":"test-trainjob-5pj7c","reconcileID":"0f528e08-194c-472f-b47e-8688564476cc","job":"test-ns-vhmn6/test-trainjob-5pj7c","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-vhmn6"}
{"level":"Level(-2)","ts":"2026-11-02T22:11:00.298141096Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-vhmn6","name":"test-trainjob-5pj7c","reconcileID":"e10616ad-7380-4857-adf1-2f99ed2b246c","job":"test-ns-vhmn6/test-trainjob-5pj7c","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-vhmn6"}
{"level":"Level(-2)","ts":"2026-11-02T22:11:00.342768003Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-vhmn6","name":"test-trainjob-5pj7c","reconcileID":"708fee5f-45c8-48d7-a715-b59c63140224","job":"test-ns-vhmn6/test-trainjob-5pj7c","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-vhmn6"}
{"level":"Level(-2)","ts":"2026-11-02T22:11:00.40718297Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-vhmn6","name":"test-trainjob-5pj7c","reconcileID":"0e755854-1d72-4f41-8b59-f460061f8ec9","job":"test-ns-vhmn6/test-trainjob-5pj7c","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-vhmn6"}
{"level":"Level(-2)","ts":"2026-11-02T22:11:17.079640204Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-m8ndd","name":"ufbd3984a5e5","reconcileID":"df04778c-d663-4136-b54c-d4dd990902ee","job":"test-ns-m8ndd/ufbd3984a5e5","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-m8ndd"}
{"level":"Level(-2)","ts":"2026-11-02T22:11:17.535363397Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-m8ndd","name":"ufbd3984a5e5","reconcileID":"1d259b5d-b11a-4165-91a2-ad3f4135c979","job":"test-ns-m8ndd/ufbd3984a5e5","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-m8ndd"}
{"level":"Level(-2)","ts":"2026-11-02T22:11:17.755421996Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-m8ndd","name":"ufbd3984a5e5","reconcileID":"09976ab5-17b1-48ab-bf1c-649b2c051104","job":"test-ns-m8ndd/ufbd3984a5e5","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-m8ndd"}
{"level":"Level(-2)","ts":"2026-11-02T22:11:17.866446661Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-m8ndd","name":"ufbd3984a5e5","reconcileID":"f0845b62-7ebe-4c4f-9412-ac6ac11487c6","job":"test-ns-m8ndd/ufbd3984a5e5","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-m8ndd"}
{"level":"Level(-2)","ts":"2026-11-02T22:11:17.949952009Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-m8ndd","name":"ufbd3984a5e5","reconcileID":"00b72dd2-2707-4a11-9404-9bb50d8f4a7b","job":"test-ns-m8ndd/ufbd3984a5e5","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-m8ndd"}
{"level":"Level(-2)","ts":"2026-11-02T22:11:18.060206838Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-m8ndd","name":"ufbd3984a5e5","reconcileID":"8e5b3949-022a-49c4-b526-9d18e5e19abd","job":"test-ns-m8ndd/ufbd3984a5e5","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-m8ndd"}
{"level":"Level(-2)","ts":"2026-11-02T22:11:19.918089254Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-m8ndd","name":"ufbd3984a5e5","reconcileID":"ca38b327-c644-494f-978f-6f3f52603928","job":"test-ns-m8ndd/ufbd3984a5e5","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-m8ndd"}
{"level":"Level(-2)","ts":"2026-11-02T22:11:20.028195338Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-m8ndd","name":"ufbd3984a5e5","reconcileID":"15bfcd48-ba73-4d5b-a637-96b28029557e","job":"test-ns-m8ndd/ufbd3984a5e5","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-m8ndd"}
{"level":"Level(-2)","ts":"2026-11-02T22:11:51.824356685Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-m8ndd","name":"ufbd3984a5e5","reconcileID":"725f817b-0b6f-4813-b84e-c087fb6c57bd","job":"test-ns-m8ndd/ufbd3984a5e5","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-m8ndd"}
{"level":"Level(-2)","ts":"2026-11-02T22:11:51.946279277Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-m8ndd","name":"ufbd3984a5e5","reconcileID":"b6bf047c-12ad-4f3d-bd06-bab584641cec","job":"test-ns-m8ndd/ufbd3984a5e5","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-m8ndd"}
{"level":"Level(-2)","ts":"2026-11-02T22:11:52.835308491Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-m8ndd","name":"ufbd3984a5e5","reconcileID":"d432e572-d480-47f9-9743-aaa69874e488","job":"test-ns-m8ndd/ufbd3984a5e5","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-m8ndd"}
{"level":"Level(-2)","ts":"2026-11-02T22:11:52.943782993Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-m8ndd","name":"ufbd3984a5e5","reconcileID":"10266d4a-befa-4ba5-bfce-aca09462d50e","job":"test-ns-m8ndd/ufbd3984a5e5","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-m8ndd"}
{"level":"Level(-2)","ts":"2026-11-02T22:11:52.985890684Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-m8ndd","name":"ufbd3984a5e5","reconcileID":"57a7080c-aa85-4f9e-b091-ca07713702e0","job":"test-ns-m8ndd/ufbd3984a5e5","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-m8ndd"}
{"level":"Level(-2)","ts":"2026-11-02T22:11:53.052561355Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-m8ndd","name":"ufbd3984a5e5","reconcileID":"354c46e3-07d6-4d10-8230-6050f776179d","job":"test-ns-m8ndd/ufbd3984a5e5","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-m8ndd"}
{"level":"Level(-2)","ts":"2026-11-02T22:11:57.913927432Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:141","msg":"ResourceFlavor create event","replica-role":"leader","resourceFlavor":{"name":"rf-zl9rz"}}
{"level":"Level(-2)","ts":"2026-11-02T22:11:57.914114249Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-zl9rz","reconcileID":"6a59ef40-693a-4f55-8762-c7f3fe43199d"}
{"level":"Level(-2)","ts":"2026-11-02T22:11:58.251484949Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:351","msg":"ClusterQueue create event","replica-role":"leader","clusterQueue":{"name":"cq-w68m7"}}
{"level":"Level(-2)","ts":"2026-11-02T22:11:58.251744822Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-w68m7","reconcileID":"3fcc190b-5558-4eb5-88a0-aa795e0a5dff"}
{"level":"Level(-2)","ts":"2026-11-02T22:11:58.267643691Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:240","msg":"LocalQueue create event","replica-role":"leader","localQueue":{"name":"lq-tcvvx","namespace":"test-ns-x267j"}}
{"level":"Level(-2)","ts":"2026-11-02T22:11:58.26781171Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-x267j","name":"lq-tcvvx","reconcileID":"9b53c24b-f253-4eb3-881a-6d917a1fd985"}
{"level":"Level(-2)","ts":"2026-11-02T22:11:58.308724594Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-w68m7"}}
{"level":"Level(-2)","ts":"2026-11-02T22:11:58.309105115Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-w68m7","reconcileID":"44639f62-dc6e-4078-8ab2-c5772bf4d73f"}
{"level":"Level(-2)","ts":"2026-11-02T22:11:58.321930057Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-x267j","name":"lq-tcvvx","reconcileID":"244cec03-7455-4cfd-a6c9-ae98818c8d35"}
{"level":"Level(-2)","ts":"2026-11-02T22:11:58.321996455Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-tcvvx","namespace":"test-ns-x267j"}}
{"level":"Level(-3)","ts":"2026-11-02T22:11:58.375003775Z","logger":"localqueue-reconciler","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-x267j","name":"lq-tcvvx","reconcileID":"244cec03-7455-4cfd-a6c9-ae98818c8d35","error":"Operation cannot be fulfilled on localqueues.kueue.x-k8s.io \"lq-tcvvx\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-11-02T22:11:58.375072996Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-x267j","name":"lq-tcvvx","reconcileID":"a7abec56-8adc-4063-b5dc-ddb4563bb760"}
{"level":"Level(-2)","ts":"2026-11-02T22:11:58.431268195Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-tcvvx","namespace":"test-ns-x267j"}}
{"level":"Level(-2)","ts":"2026-11-02T22:11:58.4313647Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-x267j","name":"lq-tcvvx","reconcileID":"7f7d5db9-74f6-4f12-8263-a3f821adfbff"}
{"level":"Level(-2)","ts":"2026-11-02T22:11:59.041507245Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-x267j","name":"dw-deployment-nvpnq-7c99cd8c4f-k27zm","reconcileID":"9a95c58b-c921-4f78-9728-9d96cf029d8d","job":"test-ns-x267j/dw-deployment-nvpnq-7c99cd8c4f-k27zm","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-02T22:11:59.093416001Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1.PriorityClass","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"debug","ts":"2026-11-02T22:11:59.525255309Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"dw-deployment-nvpnq-7c99cd8c4f-k27zm","namespace":"test-ns-x267j"},"kind":"Pod","apiVersion":"v1","type":"Normal","reason":"CreatedWorkload","action":"CreatedWorkload","note":"Created Workload: test-ns-x267j/pod-dw-deployment-nvpnq-7c99cd8c4f-k27zm-edce3"}
{"level":"Level(-2)","ts":"2026-11-02T22:11:59.575056061Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1276","msg":"Workload create event","replica-role":"leader","workload":{"name":"pod-dw-deployment-nvpnq-7c99cd8c4f-k27zm-edce3","namespace":"test-ns-x267j"},"queue":"lq-tcvvx","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-02T22:11:59.575100465Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-x267j","name":"pod-dw-deployment-nvpnq-7c99cd8c4f-k27zm-edce3","reconcileID":"a6109b0f-817a-436f-a0df-afb2f7a9c917"}
{"level":"Level(-2)","ts":"2026-11-02T22:11:59.575133968Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-x267j","name":"dw-deployment-nvpnq-7c99cd8c4f-k27zm","reconcileID":"0c7a1a48-dcf3-47e0-ba12-99453ad741a4","job":"test-ns-x267j/dw-deployment-nvpnq-7c99cd8c4f-k27zm","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-02T22:11:59.575152636Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-x267j","name":"pod-dw-deployment-nvpnq-7c99cd8c4f-k27zm-edce3","reconcileID":"a6109b0f-817a-436f-a0df-afb2f7a9c917","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T22:11:59.575311308Z","logger":"scheduler","caller":"scheduler/scheduler.go:308","msg":"Obtained heads","replica-role":"follower","schedulingCycle":1,"headCount":1,"waitDuration":1728.722583435}
{"level":"Level(-2)","ts":"2026-11-02T22:11:59.575357711Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-x267j","name":"pod-dw-deployment-nvpnq-7c99cd8c4f-k27zm-edce3","reconcileID":"cbfb3a27-c366-4b5e-8c20-ac7b36c0180b"}
{"level":"Level(-2)","ts":"2026-11-02T22:11:59.575361265Z","logger":"scheduler","caller":"scheduler/scheduler.go:323","msg":"Snapshot taken","replica-role":"follower","schedulingCycle":1,"duration":0.000013895}
{"level":"Level(-2)","ts":"2026-11-02T22:11:59.575478876Z","logger":"scheduler","caller":"scheduler/scheduler.go:328","msg":"Nomination done","replica-role":"follower","schedulingCycle":1,"entries":1,"inadmissibleEntries":0,"duration":0.00010307}
{"level":"Level(-2)","ts":"2026-11-02T22:11:59.575503779Z","logger":"scheduler","caller":"scheduler/scheduler.go:406","msg":"Attempting to schedule workload","replica-role":"follower","schedulingCycle":1,"workload":{"name":"pod-dw-deployment-nvpnq-7c99cd8c4f-k27zm-edce3","namespace":"test-ns-x267j"},"clusterQueue":{"name":"cq-w68m7"}}
{"level":"Level(-2)","ts":"2026-11-02T22:11:59.575604752Z","logger":"scheduler","caller":"scheduler/scheduler.go:1041","msg":"Workload assumed in the cache","replica-role":"follower","schedulingCycle":1,"workload":{"name":"pod-dw-deployment-nvpnq-7c99cd8c4f-k27zm-edce3","namespace":"test-ns-x267j"},"clusterQueue":{"name":"cq-w68m7"}}
{"level":"Level(-2)","ts":"2026-11-02T22:11:59.575646976Z","logger":"scheduler","caller":"scheduler/scheduler.go:362","msg":"Workload processing done","replica-role":"follower","schedulingCycle":1,"duration":0.00014875}
{"level":"Level(-2)","ts":"2026-11-02T22:11:59.575673206Z","logger":"scheduler","caller":"scheduler/scheduler.go:297","msg":"Scheduling cycle complete","replica-role":"follower","schedulingCycle":1,"duration":1728.722946316}
{"level":"Level(-2)","ts":"2026-11-02T22:11:59.575696873Z","logger":"scheduler","caller":"scheduler/scheduler.go:295","msg":"Scheduling cycle starts","replica-role":"leader","schedulingCycle":2}
{"level":"Level(-2)","ts":"2026-11-02T22:11:59.689522553Z","logger":"scheduler","caller":"scheduler/scheduler.go:1000","msg":"Workload successfully admitted and assigned flavors","replica-role":"follower","schedulingCycle":1,"workload":{"name":"pod-dw-deployment-nvpnq-7c99cd8c4f-k27zm-edce3","namespace":"test-ns-x267j"},"clusterQueue":{"name":"cq-w68m7"},"assignments":[{"name":"main","flavors":{"cpu":"rf-zl9rz","memory":"rf-zl9rz"},"resourceUsage":{"cpu":"1","memory":"3Gi"},"count":1}]}
{"level":"debug","ts":"2026-11-02T22:11:59.689561562Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"pod-dw-deployment-nvpnq-7c99cd8c4f-k27zm-edce3","namespace":"test-ns-x267j"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"Admitted","action":"Admitted","note":"Admitted by ClusterQueue cq-w68m7, wait time since reservation was 1s"}
{"level":"debug","ts":"2026-11-02T22:11:59.6895846Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"pod-dw-deployment-nvpnq-7c99cd8c4f-k27zm-edce3","namespace":"test-ns-x267j"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"QuotaReserved","action":"QuotaReserved","note":"Quota reserved in ClusterQueue cq-w68m7, wait time since queued was 1s"}
{"level":"Level(-2)","ts":"2026-11-02T22:11:59.689733225Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"pod-dw-deployment-nvpnq-7c99cd8c4f-k27zm-edce3","namespace":"test-ns-x267j"},"queue":"lq-tcvvx","status":"admitted","prevStatus":"pending","clusterQueue":"cq-w68m7"}
{"level":"Level(-2)","ts":"2026-11-02T22:11:59.689815756Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-x267j","name":"dw-deployment-nvpnq-7c99cd8c4f-k27zm","reconcileID":"aa08ff22-ac5e-4845-9c2e-09bb7a06d010","job":"test-ns-x267j/dw-deployment-nvpnq-7c99cd8c4f-k27zm","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-02T22:11:59.689783887Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-x267j","name":"pod-dw-deployment-nvpnq-7c99cd8c4f-k27zm-edce3","reconcileID":"1cb73b4d-c6a0-4486-b404-9dc36d746dfd"}
{"level":"Level(-2)","ts":"2026-11-02T22:11:59.689947133Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-x267j","name":"pod-dw-deployment-nvpnq-7c99cd8c4f-k27zm-edce3","reconcileID":"1cb73b4d-c6a0-4486-b404-9dc36d746dfd","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T22:11:59.690018321Z","logger":"v1_pod","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-x267j","name":"dw-deployment-nvpnq-7c99cd8c4f-k27zm","reconcileID":"aa08ff22-ac5e-4845-9c2e-09bb7a06d010","job":"test-ns-x267j/dw-deployment-nvpnq-7c99cd8c4f-k27zm","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-02T22:11:59.690081808Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-x267j","name":"pod-dw-deployment-nvpnq-7c99cd8c4f-k27zm-edce3","reconcileID":"885c779a-a1e0-4a84-8a9b-e4d2d4152056"}
{"level":"debug","ts":"2026-11-02T22:11:59.818409836Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"dw-deployment-nvpnq-7c99cd8c4f-k27zm","namespace":"test-ns-x267j"},"kind":"Pod","apiVersion":"v1","type":"Normal","reason":"Started","action":"Started","note":"Admitted by clusterQueue cq-w68m7"}
{"level":"Level(-2)","ts":"2026-11-02T22:11:59.818840732Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-x267j","name":"dw-deployment-nvpnq-7c99cd8c4f-k27zm","reconcileID":"aaf1691a-9611-49ce-8478-ca2f18ce2a7b","job":"test-ns-x267j/dw-deployment-nvpnq-7c99cd8c4f-k27zm","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-02T22:11:59.825688312Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-x267j","name":"dw-deployment-nvpnq-7c99cd8c4f-k27zm","reconcileID":"8d7c72e1-9025-4ce2-927f-22d404d21f06","job":"test-ns-x267j/dw-deployment-nvpnq-7c99cd8c4f-k27zm","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-02T22:11:59.886242434Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-x267j","name":"dw-deployment-nvpnq-7c99cd8c4f-k27zm","reconcileID":"49c50141-13bc-4b10-9781-ded964506ccd","job":"test-ns-x267j/dw-deployment-nvpnq-7c99cd8c4f-k27zm","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-02T22:11:59.992714481Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-x267j","name":"dw-deployment-nvpnq-7c99cd8c4f-k27zm","reconcileID":"6e46e1d7-b515-4d5b-b157-c5a2cfaad3f0","job":"test-ns-x267j/dw-deployment-nvpnq-7c99cd8c4f-k27zm","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-02T22:12:00.575617136Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-w68m7","reconcileID":"12c6b283-9de7-47a6-9aa8-172b955be672"}
{"level":"Level(-2)","ts":"2026-11-02T22:12:00.575617116Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-x267j","name":"lq-tcvvx","reconcileID":"a98dabc4-703e-4c76-ae14-15daa224682f"}
{"level":"Level(-2)","ts":"2026-11-02T22:12:00.632473008Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-tcvvx","namespace":"test-ns-x267j"}}
{"level":"Level(-2)","ts":"2026-11-02T22:12:00.63262471Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-x267j","name":"lq-tcvvx","reconcileID":"e4a8904e-72d9-4fb6-82aa-090af3ceb79b"}
{"level":"Level(-2)","ts":"2026-11-02T22:12:00.635665068Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-w68m7"}}
{"level":"Level(-2)","ts":"2026-11-02T22:12:00.635830927Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-w68m7","reconcileID":"e6805b8b-aa4d-43cb-bfe2-e56095a22594"}
{"level":"Level(-2)","ts":"2026-11-02T22:12:00.810395215Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-x267j","name":"dw-deployment-nvpnq-7c99cd8c4f-k27zm","reconcileID":"ca3c27d1-88b0-429e-a781-33527a69b050","job":"test-ns-x267j/dw-deployment-nvpnq-7c99cd8c4f-k27zm","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-02T22:12:00.819222384Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-x267j","name":"pod-dw-deployment-nvpnq-7c99cd8c4f-k27zm-edce3","reconcileID":"e97b8314-e509-4e3c-9441-5034584fbb19"}
{"level":"Level(-2)","ts":"2026-11-02T22:12:02.766109732Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-x267j","name":"dw-deployment-nvpnq-7c99cd8c4f-k27zm","reconcileID":"e616ee40-89c7-4b77-9fbf-8708fb4dce00","job":"test-ns-x267j/dw-deployment-nvpnq-7c99cd8c4f-k27zm","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-02T22:12:03.766500666Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-x267j","name":"pod-dw-deployment-nvpnq-7c99cd8c4f-k27zm-edce3","reconcileID":"3642d333-20c6-4d81-92f4-41c3d3b54fe4"}
{"level":"info","ts":"2026-11-02T22:12:14.921619512Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:117","msg":"Validating create","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"h7c9ae6b04b1","namespace":"test-ns-x267j"},"namespace":"test-ns-x267j","name":"h7c9ae6b04b1","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:redhat-ods-applications:trainer-e2e","requestID":"dc16f405-a779-4245-bdf5-9069e48914f2","replica-role":"leader"}
{"level":"Level(-2)","ts":"2026-11-02T22:12:14.980662583Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-x267j","name":"h7c9ae6b04b1","reconcileID":"96fa007e-e0ba-457f-9404-0531b777d658","job":"test-ns-x267j/h7c9ae6b04b1","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:12:15.298197009Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-x267j","name":"h7c9ae6b04b1","reconcileID":"3adf5f72-61cb-46d5-8187-5ad6ba37944a","job":"test-ns-x267j/h7c9ae6b04b1","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"debug","ts":"2026-11-02T22:12:15.298199635Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"h7c9ae6b04b1","namespace":"test-ns-x267j"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"CreatedWorkload","action":"CreatedWorkload","note":"Created Workload: test-ns-x267j/trainjob-h7c9ae6b04b1-15263"}
{"level":"Level(-2)","ts":"2026-11-02T22:12:15.298613783Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1276","msg":"Workload create event","replica-role":"leader","workload":{"name":"trainjob-h7c9ae6b04b1-15263","namespace":"test-ns-x267j"},"queue":"lq-tcvvx","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-02T22:12:15.298673744Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-x267j","name":"trainjob-h7c9ae6b04b1-15263","reconcileID":"5bf943fe-2a52-41bf-8bc7-950f92f9843d"}
{"level":"Level(-2)","ts":"2026-11-02T22:12:15.298727916Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-x267j","name":"trainjob-h7c9ae6b04b1-15263","reconcileID":"5bf943fe-2a52-41bf-8bc7-950f92f9843d","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T22:12:15.298794697Z","logger":"scheduler","caller":"scheduler/scheduler.go:308","msg":"Obtained heads","replica-role":"leader","schedulingCycle":2,"headCount":1,"waitDuration":15.72309736}
{"level":"Level(-2)","ts":"2026-11-02T22:12:15.298833335Z","logger":"scheduler","caller":"scheduler/scheduler.go:323","msg":"Snapshot taken","replica-role":"leader","schedulingCycle":2,"duration":0.000010249}
{"level":"Level(-2)","ts":"2026-11-02T22:12:15.298859676Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-x267j","name":"trainjob-h7c9ae6b04b1-15263","reconcileID":"a1f59104-dc03-480d-a106-fb75d1b03eb0"}
{"level":"Level(-2)","ts":"2026-11-02T22:12:15.29890843Z","logger":"scheduler","caller":"scheduler/scheduler.go:328","msg":"Nomination done","replica-role":"leader","schedulingCycle":2,"entries":1,"inadmissibleEntries":0,"duration":0.000066204}
{"level":"Level(-2)","ts":"2026-11-02T22:12:15.298926708Z","logger":"scheduler","caller":"scheduler/scheduler.go:406","msg":"Attempting to schedule workload","replica-role":"leader","schedulingCycle":2,"workload":{"name":"trainjob-h7c9ae6b04b1-15263","namespace":"test-ns-x267j"},"clusterQueue":{"name":"cq-w68m7"}}
{"level":"Level(-2)","ts":"2026-11-02T22:12:15.299030404Z","logger":"scheduler","caller":"scheduler/scheduler.go:1041","msg":"Workload assumed in the cache","replica-role":"leader","schedulingCycle":2,"workload":{"name":"trainjob-h7c9ae6b04b1-15263","namespace":"test-ns-x267j"},"clusterQueue":{"name":"cq-w68m7"}}
{"level":"Level(-2)","ts":"2026-11-02T22:12:15.299053489Z","logger":"scheduler","caller":"scheduler/scheduler.go:362","msg":"Workload processing done","replica-role":"leader","schedulingCycle":2,"duration":0.000128709}
{"level":"Level(-2)","ts":"2026-11-02T22:12:15.299065564Z","logger":"scheduler","caller":"scheduler/scheduler.go:297","msg":"Scheduling cycle complete","replica-role":"leader","schedulingCycle":2,"duration":15.723368805}
{"level":"Level(-2)","ts":"2026-11-02T22:12:15.299079711Z","logger":"scheduler","caller":"scheduler/scheduler.go:295","msg":"Scheduling cycle starts","replica-role":"leader","schedulingCycle":3}
{"level":"Level(-2)","ts":"2026-11-02T22:12:15.410703526Z","logger":"scheduler","caller":"scheduler/scheduler.go:1000","msg":"Workload successfully admitted and assigned flavors","replica-role":"leader","schedulingCycle":2,"workload":{"name":"trainjob-h7c9ae6b04b1-15263","namespace":"test-ns-x267j"},"clusterQueue":{"name":"cq-w68m7"},"assignments":[{"name":"node","flavors":{"cpu":"rf-zl9rz","memory":"rf-zl9rz"},"resourceUsage":{"cpu":"4","memory":"16Gi"},"count":2}]}
{"level":"debug","ts":"2026-11-02T22:12:15.410761536Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-h7c9ae6b04b1-15263","namespace":"test-ns-x267j"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"QuotaReserved","action":"QuotaReserved","note":"Quota reserved in ClusterQueue cq-w68m7, wait time since queued was 0s"}
{"level":"debug","ts":"2026-11-02T22:12:15.41080775Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-h7c9ae6b04b1-15263","namespace":"test-ns-x267j"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"Admitted","action":"Admitted","note":"Admitted by ClusterQueue cq-w68m7, wait time since reservation was 0s"}
{"level":"Level(-2)","ts":"2026-11-02T22:12:15.411042375Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-h7c9ae6b04b1-15263","namespace":"test-ns-x267j"},"queue":"lq-tcvvx","status":"admitted","prevStatus":"pending","clusterQueue":"cq-w68m7"}
{"level":"Level(-2)","ts":"2026-11-02T22:12:15.411044352Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-x267j","name":"trainjob-h7c9ae6b04b1-15263","reconcileID":"a8bbf175-c549-4762-9f65-1ed122895afa"}
{"level":"Level(-2)","ts":"2026-11-02T22:12:15.411093656Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-x267j","name":"trainjob-h7c9ae6b04b1-15263","reconcileID":"a8bbf175-c549-4762-9f65-1ed122895afa","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T22:12:15.411283371Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-x267j","name":"trainjob-h7c9ae6b04b1-15263","reconcileID":"3ff878fb-3db2-4d97-b8ea-a50dbda4d048"}
{"level":"error","ts":"2026-11-02T22:12:15.617766436Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-x267j","name":"h7c9ae6b04b1","reconcileID":"3adf5f72-61cb-46d5-8187-5ad6ba37944a","error":"workloads.kueue.x-k8s.io \"trainjob-h7c9ae6b04b1-15263\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T22:12:15.617871118Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-x267j","name":"h7c9ae6b04b1","reconcileID":"85d8f756-e14c-435e-b8ed-c91ed95e1ce1","job":"test-ns-x267j/h7c9ae6b04b1","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:12:15.773924318Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-x267j","name":"h7c9ae6b04b1","reconcileID":"85d8f756-e14c-435e-b8ed-c91ed95e1ce1","job":"test-ns-x267j/h7c9ae6b04b1","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-3)","ts":"2026-11-02T22:12:15.827724343Z","logger":"trainjob","caller":"jobframework/reconciler.go:594","msg":"Unsuspending job","replica-role":"leader","namespace":"test-ns-x267j","name":"h7c9ae6b04b1","reconcileID":"85d8f756-e14c-435e-b8ed-c91ed95e1ce1","job":"test-ns-x267j/h7c9ae6b04b1","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"h7c9ae6b04b1\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-3)","ts":"2026-11-02T22:12:15.827785029Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-x267j","name":"h7c9ae6b04b1","reconcileID":"85d8f756-e14c-435e-b8ed-c91ed95e1ce1","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"h7c9ae6b04b1\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-11-02T22:12:15.827868812Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-x267j","name":"h7c9ae6b04b1","reconcileID":"443a7929-a967-4dd4-8559-a0ba4567a27f","job":"test-ns-x267j/h7c9ae6b04b1","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:12:15.984510426Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-x267j","name":"h7c9ae6b04b1","reconcileID":"443a7929-a967-4dd4-8559-a0ba4567a27f","job":"test-ns-x267j/h7c9ae6b04b1","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"info","ts":"2026-11-02T22:12:16.040448855Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"h7c9ae6b04b1","namespace":"test-ns-x267j"},"namespace":"test-ns-x267j","name":"h7c9ae6b04b1","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"3d6e22da-8af5-4f76-bb34-ffe3167dc226","replica-role":"leader"}
{"level":"info","ts":"2026-11-02T22:12:16.152562983Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"h7c9ae6b04b1","namespace":"test-ns-x267j"},"namespace":"test-ns-x267j","name":"h7c9ae6b04b1","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"6afbddac-9cbc-4168-bb40-36a4210b5e04","replica-role":"leader"}
{"level":"Level(-2)","ts":"2026-11-02T22:12:16.20922657Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-x267j","name":"h7c9ae6b04b1","reconcileID":"c6d94248-29cd-4db9-8c47-c5568366204c","job":"test-ns-x267j/h7c9ae6b04b1","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"debug","ts":"2026-11-02T22:12:16.209234071Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"h7c9ae6b04b1","namespace":"test-ns-x267j"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"Started","action":"Started","note":"Admitted by clusterQueue cq-w68m7"}
{"level":"Level(-2)","ts":"2026-11-02T22:12:16.29927032Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-x267j","name":"lq-tcvvx","reconcileID":"6d03ec58-3626-4fd2-8495-96a53c567855"}
{"level":"Level(-2)","ts":"2026-11-02T22:12:16.299278837Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-w68m7","reconcileID":"ae14d4be-7455-4794-a524-8e3ac24b5f98"}
{"level":"Level(-2)","ts":"2026-11-02T22:12:16.353563823Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-tcvvx","namespace":"test-ns-x267j"}}
{"level":"Level(-2)","ts":"2026-11-02T22:12:16.353678353Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-x267j","name":"lq-tcvvx","reconcileID":"b430ee3c-b7cb-49e9-a0b8-27ae17071582"}
{"level":"Level(-2)","ts":"2026-11-02T22:12:16.356478522Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-w68m7"}}
{"level":"Level(-2)","ts":"2026-11-02T22:12:16.356627148Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-w68m7","reconcileID":"610186e3-2467-4274-8aa4-e1494824ea18"}
{"level":"Level(-2)","ts":"2026-11-02T22:12:16.366354746Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-x267j","name":"h7c9ae6b04b1","reconcileID":"c6d94248-29cd-4db9-8c47-c5568366204c","job":"test-ns-x267j/h7c9ae6b04b1","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-3)","ts":"2026-11-02T22:12:16.420324922Z","logger":"trainjob","caller":"jobframework/reconciler.go:594","msg":"Unsuspending job","replica-role":"leader","namespace":"test-ns-x267j","name":"h7c9ae6b04b1","reconcileID":"c6d94248-29cd-4db9-8c47-c5568366204c","job":"test-ns-x267j/h7c9ae6b04b1","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"h7c9ae6b04b1\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-3)","ts":"2026-11-02T22:12:16.420382361Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-x267j","name":"h7c9ae6b04b1","reconcileID":"c6d94248-29cd-4db9-8c47-c5568366204c","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"h7c9ae6b04b1\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-11-02T22:12:16.42046817Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-x267j","name":"h7c9ae6b04b1","reconcileID":"74767bcf-7397-47d8-b7cb-e1b904ce2377","job":"test-ns-x267j/h7c9ae6b04b1","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:12:16.578728414Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-x267j","name":"h7c9ae6b04b1","reconcileID":"ee3c751b-a7e9-4e8e-9842-0a408b769c23","job":"test-ns-x267j/h7c9ae6b04b1","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:12:16.668496017Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1.PartialObjectMetadata","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-02T22:12:16.735866051Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-x267j","name":"h7c9ae6b04b1","reconcileID":"3aefc5f6-d207-473c-ade2-acaa95567bca","job":"test-ns-x267j/h7c9ae6b04b1","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:12:16.771097539Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1.PartialObjectMetadata","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-02T22:12:16.893207236Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-x267j","name":"h7c9ae6b04b1","reconcileID":"5c078c1a-374d-4f1b-a4a3-d4035d4e7594","job":"test-ns-x267j/h7c9ae6b04b1","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:12:17.174556578Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-x267j","name":"h7c9ae6b04b1","reconcileID":"ea62674b-e215-4773-89b1-dff785c92b85","job":"test-ns-x267j/h7c9ae6b04b1","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:12:17.335469851Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-x267j","name":"h7c9ae6b04b1","reconcileID":"6718c219-a6a5-4d01-a689-c4635a3f4d9c","job":"test-ns-x267j/h7c9ae6b04b1","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:12:17.931266397Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-x267j","name":"trainjob-h7c9ae6b04b1-15263","reconcileID":"a19f5a5a-e3ef-4729-8c7b-3ab21540f5fa"}
{"level":"Level(-2)","ts":"2026-11-02T22:12:19.49189463Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-x267j","name":"trainjob-h7c9ae6b04b1-15263","reconcileID":"d2aec504-4ae2-4bdd-aa3b-2cdb6ae2c4b9"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:16.508191123Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-x267j","name":"trainjob-h7c9ae6b04b1-15263","reconcileID":"1325fb84-0e00-446f-bb80-18b6c9befbbe"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:16.579549302Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-x267j","name":"h7c9ae6b04b1","reconcileID":"ed217b02-becf-4adb-b698-5684dcd01b8f","job":"test-ns-x267j/h7c9ae6b04b1","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:16.74479476Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-x267j","name":"h7c9ae6b04b1","reconcileID":"31624435-1d6f-4ed8-8446-28eb5bde304a","job":"test-ns-x267j/h7c9ae6b04b1","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:43.652879062Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-x267j","name":"trainjob-h7c9ae6b04b1-15263","reconcileID":"916cd32a-d1b6-4f5d-b0f9-b0575389ceeb"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:43.67295023Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-x267j","name":"h7c9ae6b04b1","reconcileID":"f4376e3f-c766-4ca6-9341-ae6b861b122b","job":"test-ns-x267j/h7c9ae6b04b1","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:43.834667418Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-x267j","name":"h7c9ae6b04b1","reconcileID":"62365453-58d1-4f1f-bb48-98984308ad0e","job":"test-ns-x267j/h7c9ae6b04b1","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:44.684663863Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-x267j","name":"h7c9ae6b04b1","reconcileID":"4709a51c-ae94-455d-9da9-ac35974c3dc5","job":"test-ns-x267j/h7c9ae6b04b1","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:44.807159007Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-x267j","name":"trainjob-h7c9ae6b04b1-15263","reconcileID":"4a45f697-3056-4818-92cb-95d40aec72f8"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:44.842745883Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-x267j","name":"h7c9ae6b04b1","reconcileID":"f37e231a-939f-4587-b7f6-3e86cc5d099c","job":"test-ns-x267j/h7c9ae6b04b1","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:44.999875628Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-x267j","name":"h7c9ae6b04b1","reconcileID":"028f189a-f9c9-4fe9-b33f-eba68cc5cb79","job":"test-ns-x267j/h7c9ae6b04b1","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"debug","ts":"2026-11-02T22:13:45.268625249Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"h7c9ae6b04b1","namespace":"test-ns-x267j"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"FinishedWorkload","action":"FinishedWorkload","note":"Workload 'test-ns-x267j/trainjob-h7c9ae6b04b1-15263' is declared finished"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:45.268948296Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-h7c9ae6b04b1-15263","namespace":"test-ns-x267j"},"queue":"lq-tcvvx","status":"finished","prevStatus":"admitted","clusterQueue":"cq-w68m7"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:45.269052911Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-x267j","name":"trainjob-h7c9ae6b04b1-15263","reconcileID":"540df8f7-a3ae-4127-a1c0-7dad39582859"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:45.26910445Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-x267j","name":"trainjob-h7c9ae6b04b1-15263","reconcileID":"540df8f7-a3ae-4127-a1c0-7dad39582859","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T22:13:45.269170951Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-x267j","name":"trainjob-h7c9ae6b04b1-15263","reconcileID":"bd02c9d6-f010-4c86-9203-5b33d5f0ef84"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:45.269174795Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-x267j","name":"h7c9ae6b04b1","reconcileID":"ddcde407-be86-4874-9bbc-813ec947d28d","job":"test-ns-x267j/h7c9ae6b04b1","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"debug","ts":"2026-11-02T22:13:45.426428188Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"h7c9ae6b04b1","namespace":"test-ns-x267j"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"FinishedWorkload","action":"FinishedWorkload","note":"Workload 'test-ns-x267j/trainjob-h7c9ae6b04b1-15263' is declared finished"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:45.549914034Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-h7c9ae6b04b1-15263","namespace":"test-ns-x267j"},"queue":"lq-tcvvx","status":"finished","clusterQueue":"cq-w68m7"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:45.549969874Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-x267j","name":"trainjob-h7c9ae6b04b1-15263","reconcileID":"6077b318-af53-4421-8b9f-4ed26a68b010"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:45.550020767Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-x267j","name":"trainjob-h7c9ae6b04b1-15263","reconcileID":"6077b318-af53-4421-8b9f-4ed26a68b010","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T22:13:45.550064652Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-x267j","name":"h7c9ae6b04b1","reconcileID":"bb935f34-da1d-4d8f-b006-4b18b211f4ec","job":"test-ns-x267j/h7c9ae6b04b1","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:45.550090508Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-x267j","name":"trainjob-h7c9ae6b04b1-15263","reconcileID":"1e5475b1-e7ca-4468-82d9-e53a3334070b"}
{"level":"debug","ts":"2026-11-02T22:13:45.735813147Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"h7c9ae6b04b1","namespace":"test-ns-x267j"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"FinishedWorkload","action":"FinishedWorkload","note":"Workload 'test-ns-x267j/trainjob-h7c9ae6b04b1-15263' is declared finished"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:46.270144895Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-x267j","name":"lq-tcvvx","reconcileID":"06c398ce-f890-46a4-be1b-421566ff82a5"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:46.270162884Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-w68m7","reconcileID":"7cca3746-f3f7-466d-b85d-ec15b23542d4"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:46.326806905Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-tcvvx","namespace":"test-ns-x267j"}}
{"level":"Level(-2)","ts":"2026-11-02T22:13:46.326982163Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-x267j","name":"lq-tcvvx","reconcileID":"55728b29-bdd8-405a-b59a-c0d74d2bc75f"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:46.329782968Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-w68m7"}}
{"level":"Level(-2)","ts":"2026-11-02T22:13:46.329943753Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-w68m7","reconcileID":"183fe429-0585-48f6-9a37-7783180aecbd"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:47.938942972Z","logger":"trainjob","caller":"workload/workload.go:1810","msg":"Workload is orphaned; finishing to release quota","replica-role":"leader","namespace":"test-ns-x267j","name":"h7c9ae6b04b1","reconcileID":"f476ca17-6c34-4ae4-8302-1dbb870ac6ee","job":"test-ns-x267j/h7c9ae6b04b1","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:47.951683305Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1312","msg":"Workload delete event","replica-role":"leader","workload":{"name":"trainjob-h7c9ae6b04b1-15263","namespace":"test-ns-x267j"},"queue":"lq-tcvvx","status":"finished"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:47.951716246Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-x267j","name":"trainjob-h7c9ae6b04b1-15263","reconcileID":"00d8b21f-192c-457e-a50a-afb8313eff09"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:47.951761289Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-x267j","name":"trainjob-h7c9ae6b04b1-15263","reconcileID":"00d8b21f-192c-457e-a50a-afb8313eff09","isDeleted":true}
{"level":"Level(-2)","ts":"2026-11-02T22:13:48.950330004Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-w68m7"}}
{"level":"Level(-2)","ts":"2026-11-02T22:13:48.950409581Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-w68m7","reconcileID":"27047dac-e7a8-4ea2-83bc-2a4e9e18eb8f"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:48.950419534Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-x267j","name":"pod-dw-deployment-nvpnq-7c99cd8c4f-k27zm-edce3","reconcileID":"7ecb7f69-30ab-4f4a-b447-47ec04796c28"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:48.952133561Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-x267j","name":"lq-tcvvx","reconcileID":"1479a305-0432-4101-a7ef-0e19e0ff06d3"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:48.963068394Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:171","msg":"ResourceFlavor update event","replica-role":"leader","resourceFlavor":{"name":"rf-zl9rz"}}
{"level":"Level(-2)","ts":"2026-11-02T22:13:48.963145681Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-zl9rz","reconcileID":"78db4f67-ec98-4a79-a69d-f268ef0e3253"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:49.005355075Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-w68m7"}}
{"level":"Level(-2)","ts":"2026-11-02T22:13:49.005390226Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-x267j","name":"pod-dw-deployment-nvpnq-7c99cd8c4f-k27zm-edce3","reconcileID":"3f066e64-1ff5-4a64-bebb-53c24921cff5"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:49.005421831Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-w68m7","reconcileID":"afffeade-0e7f-49bd-b410-c85f0f01a672"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:49.005421033Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-x267j","name":"lq-tcvvx","reconcileID":"e49b1244-15a1-40bf-b821-70cb0e642fa7"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:49.0597179Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-tcvvx","namespace":"test-ns-x267j"}}
{"level":"Level(-2)","ts":"2026-11-02T22:13:49.059860614Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-x267j","name":"lq-tcvvx","reconcileID":"23ee631b-01c8-4118-a231-70091c92b7a9"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:49.487013551Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:141","msg":"ResourceFlavor create event","replica-role":"leader","resourceFlavor":{"name":"rf-bkvnn"}}
{"level":"Level(-2)","ts":"2026-11-02T22:13:49.487106382Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-bkvnn","reconcileID":"e11069ba-df90-4a7b-8057-0057f412f831"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:49.821543788Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:351","msg":"ClusterQueue create event","replica-role":"leader","clusterQueue":{"name":"cq-4rjwk"}}
{"level":"Level(-2)","ts":"2026-11-02T22:13:49.821765635Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-4rjwk","reconcileID":"f0ad976e-9b6c-4f0d-a54f-e70e55cf27d9"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:49.837272189Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:240","msg":"LocalQueue create event","replica-role":"leader","localQueue":{"name":"lq-b4tfb","namespace":"test-ns-xxbws"}}
{"level":"Level(-2)","ts":"2026-11-02T22:13:49.837370575Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-xxbws","name":"lq-b4tfb","reconcileID":"a1c596f4-7c55-495a-a587-615a2278c451"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:49.876331808Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-4rjwk"}}
{"level":"Level(-2)","ts":"2026-11-02T22:13:49.87648539Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-4rjwk","reconcileID":"691787c9-a221-46ef-a50a-8c0da68e8fa5"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:49.893179488Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-b4tfb","namespace":"test-ns-xxbws"}}
{"level":"Level(-2)","ts":"2026-11-02T22:13:49.893308165Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-xxbws","name":"lq-b4tfb","reconcileID":"852c29a6-ac8a-4164-9b92-dbeae6d0b773"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:49.934606036Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-x267j","name":"pod-dw-deployment-nvpnq-7c99cd8c4f-k27zm-edce3","reconcileID":"7d039aff-c751-4013-8b3c-90e030fa91a8"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:49.949351626Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-b4tfb","namespace":"test-ns-xxbws"}}
{"level":"Level(-2)","ts":"2026-11-02T22:13:49.949433304Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-xxbws","name":"lq-b4tfb","reconcileID":"1d33d059-e646-4af9-9012-8fc69da91966"}
{"level":"info","ts":"2026-11-02T22:13:50.120324733Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:117","msg":"Validating create","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-preemption-trainjob-gsqlf","namespace":"test-ns-xxbws"},"namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:admin","requestID":"eb5cdfb2-ce48-41fb-99fa-a4af62fdcb8d","replica-role":"leader"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:50.173179512Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"832e0373-bc93-460a-95d9-66e861a70255","job":"test-ns-xxbws/test-preemption-trainjob-gsqlf","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"debug","ts":"2026-11-02T22:13:50.488031134Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"test-preemption-trainjob-gsqlf","namespace":"test-ns-xxbws"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"CreatedWorkload","action":"CreatedWorkload","note":"Created Workload: test-ns-xxbws/trainjob-test-preemption-trainjob-gsqlf-ffca7"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:50.488198144Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1276","msg":"Workload create event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-gsqlf-ffca7","namespace":"test-ns-xxbws"},"queue":"lq-b4tfb","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:50.48824511Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-xxbws","name":"trainjob-test-preemption-trainjob-gsqlf-ffca7","reconcileID":"7c7d6596-493a-4c77-add8-9e9361c03d33"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:50.488291769Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-xxbws","name":"trainjob-test-preemption-trainjob-gsqlf-ffca7","reconcileID":"7c7d6596-493a-4c77-add8-9e9361c03d33","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T22:13:50.488355657Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"f9ed23a9-9184-42b5-8c6e-c4376047f64d","job":"test-ns-xxbws/test-preemption-trainjob-gsqlf","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:50.48840604Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-xxbws","name":"trainjob-test-preemption-trainjob-gsqlf-ffca7","reconcileID":"a78084c3-97e3-4707-898b-bf273806b1e0"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:50.488425183Z","logger":"scheduler","caller":"scheduler/scheduler.go:308","msg":"Obtained heads","replica-role":"leader","schedulingCycle":3,"headCount":1,"waitDuration":95.18934467}
{"level":"Level(-2)","ts":"2026-11-02T22:13:50.488468717Z","logger":"scheduler","caller":"scheduler/scheduler.go:323","msg":"Snapshot taken","replica-role":"leader","schedulingCycle":3,"duration":0.000015293}
{"level":"Level(-2)","ts":"2026-11-02T22:13:50.488548234Z","logger":"scheduler","caller":"scheduler/scheduler.go:328","msg":"Nomination done","replica-role":"leader","schedulingCycle":3,"entries":1,"inadmissibleEntries":0,"duration":0.000065913}
{"level":"Level(-2)","ts":"2026-11-02T22:13:50.488570331Z","logger":"scheduler","caller":"scheduler/scheduler.go:406","msg":"Attempting to schedule workload","replica-role":"leader","schedulingCycle":3,"workload":{"name":"trainjob-test-preemption-trainjob-gsqlf-ffca7","namespace":"test-ns-xxbws"},"clusterQueue":{"name":"cq-4rjwk"}}
{"level":"Level(-2)","ts":"2026-11-02T22:13:50.488639379Z","logger":"scheduler","caller":"scheduler/scheduler.go:1041","msg":"Workload assumed in the cache","replica-role":"leader","schedulingCycle":3,"workload":{"name":"trainjob-test-preemption-trainjob-gsqlf-ffca7","namespace":"test-ns-xxbws"},"clusterQueue":{"name":"cq-4rjwk"}}
{"level":"Level(-2)","ts":"2026-11-02T22:13:50.488658728Z","logger":"scheduler","caller":"scheduler/scheduler.go:362","msg":"Workload processing done","replica-role":"leader","schedulingCycle":3,"duration":0.000093202}
{"level":"Level(-2)","ts":"2026-11-02T22:13:50.488669095Z","logger":"scheduler","caller":"scheduler/scheduler.go:297","msg":"Scheduling cycle complete","replica-role":"leader","schedulingCycle":3,"duration":95.189589253}
{"level":"Level(-2)","ts":"2026-11-02T22:13:50.488684037Z","logger":"scheduler","caller":"scheduler/scheduler.go:295","msg":"Scheduling cycle starts","replica-role":"leader","schedulingCycle":4}
{"level":"Level(-2)","ts":"2026-11-02T22:13:50.597377795Z","logger":"scheduler","caller":"scheduler/scheduler.go:1000","msg":"Workload successfully admitted and assigned flavors","replica-role":"leader","schedulingCycle":3,"workload":{"name":"trainjob-test-preemption-trainjob-gsqlf-ffca7","namespace":"test-ns-xxbws"},"clusterQueue":{"name":"cq-4rjwk"},"assignments":[{"name":"node","count":1}]}
{"level":"debug","ts":"2026-11-02T22:13:50.597414809Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-test-preemption-trainjob-gsqlf-ffca7","namespace":"test-ns-xxbws"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"QuotaReserved","action":"QuotaReserved","note":"Quota reserved in ClusterQueue cq-4rjwk, wait time since queued was 1s"}
{"level":"debug","ts":"2026-11-02T22:13:50.597448026Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-test-preemption-trainjob-gsqlf-ffca7","namespace":"test-ns-xxbws"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"Admitted","action":"Admitted","note":"Admitted by ClusterQueue cq-4rjwk, wait time since reservation was 1s"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:50.597724095Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-gsqlf-ffca7","namespace":"test-ns-xxbws"},"queue":"lq-b4tfb","status":"admitted","prevStatus":"pending","clusterQueue":"cq-4rjwk"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:50.597772514Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-xxbws","name":"trainjob-test-preemption-trainjob-gsqlf-ffca7","reconcileID":"387fb458-3942-4627-9e5a-08d8ce57f736"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:50.59783014Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-xxbws","name":"trainjob-test-preemption-trainjob-gsqlf-ffca7","reconcileID":"387fb458-3942-4627-9e5a-08d8ce57f736","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T22:13:50.597882515Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-xxbws","name":"trainjob-test-preemption-trainjob-gsqlf-ffca7","reconcileID":"c6830f70-8fd4-46f5-80f1-4801e4c85896"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:50.645246872Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"691eafee-d46c-428a-bb1a-249d90fbfa4f","job":"test-ns-xxbws/test-preemption-trainjob-gsqlf","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:50.80143376Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"691eafee-d46c-428a-bb1a-249d90fbfa4f","job":"test-ns-xxbws/test-preemption-trainjob-gsqlf","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-3)","ts":"2026-11-02T22:13:50.854268031Z","logger":"trainjob","caller":"jobframework/reconciler.go:594","msg":"Unsuspending job","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"691eafee-d46c-428a-bb1a-249d90fbfa4f","job":"test-ns-xxbws/test-preemption-trainjob-gsqlf","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-preemption-trainjob-gsqlf\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-3)","ts":"2026-11-02T22:13:50.854338416Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"691eafee-d46c-428a-bb1a-249d90fbfa4f","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-preemption-trainjob-gsqlf\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:50.854443525Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"41dd4dd6-cded-41f4-aaa0-4ce8a76eea85","job":"test-ns-xxbws/test-preemption-trainjob-gsqlf","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:51.009915178Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"41dd4dd6-cded-41f4-aaa0-4ce8a76eea85","job":"test-ns-xxbws/test-preemption-trainjob-gsqlf","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-3)","ts":"2026-11-02T22:13:51.062840055Z","logger":"trainjob","caller":"jobframework/reconciler.go:594","msg":"Unsuspending job","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"41dd4dd6-cded-41f4-aaa0-4ce8a76eea85","job":"test-ns-xxbws/test-preemption-trainjob-gsqlf","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-preemption-trainjob-gsqlf\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-3)","ts":"2026-11-02T22:13:51.062920743Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"41dd4dd6-cded-41f4-aaa0-4ce8a76eea85","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-preemption-trainjob-gsqlf\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:51.063038465Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"93aff155-024d-48ef-8abc-b60e819291ac","job":"test-ns-xxbws/test-preemption-trainjob-gsqlf","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:51.219066228Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"93aff155-024d-48ef-8abc-b60e819291ac","job":"test-ns-xxbws/test-preemption-trainjob-gsqlf","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-3)","ts":"2026-11-02T22:13:51.271525869Z","logger":"trainjob","caller":"jobframework/reconciler.go:594","msg":"Unsuspending job","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"93aff155-024d-48ef-8abc-b60e819291ac","job":"test-ns-xxbws/test-preemption-trainjob-gsqlf","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-preemption-trainjob-gsqlf\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-3)","ts":"2026-11-02T22:13:51.271590024Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"93aff155-024d-48ef-8abc-b60e819291ac","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-preemption-trainjob-gsqlf\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:51.271673393Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"15a72734-677d-4dcc-9ba2-a3032d9c56b8","job":"test-ns-xxbws/test-preemption-trainjob-gsqlf","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:51.427225833Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"15a72734-677d-4dcc-9ba2-a3032d9c56b8","job":"test-ns-xxbws/test-preemption-trainjob-gsqlf","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"info","ts":"2026-11-02T22:13:51.48184752Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-preemption-trainjob-gsqlf","namespace":"test-ns-xxbws"},"namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"d01e1a07-a8e9-4b8d-a569-1b07090e7a1f","replica-role":"leader"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:51.4887388Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-4rjwk","reconcileID":"26a13883-91bc-4f4b-bd01-0087058a77fd"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:51.488735998Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-xxbws","name":"lq-b4tfb","reconcileID":"add1cf2a-6b46-4095-bb6a-0c4978f41690"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:51.54483994Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-b4tfb","namespace":"test-ns-xxbws"}}
{"level":"Level(-2)","ts":"2026-11-02T22:13:51.544950758Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-xxbws","name":"lq-b4tfb","reconcileID":"4dc6ab89-fbce-4d1e-b8f7-c83862ae1ef0"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:51.546090218Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-4rjwk"}}
{"level":"Level(-2)","ts":"2026-11-02T22:13:51.546209858Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-4rjwk","reconcileID":"677c9049-1263-403b-aef1-393e1dd54051"}
{"level":"info","ts":"2026-11-02T22:13:51.592094033Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-preemption-trainjob-gsqlf","namespace":"test-ns-xxbws"},"namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"ae43153a-c4b6-4379-a349-d5e1625dedb1","replica-role":"leader"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:51.647280602Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"fe89789c-6b96-4af7-a40c-0d44ddb44124","job":"test-ns-xxbws/test-preemption-trainjob-gsqlf","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"debug","ts":"2026-11-02T22:13:51.647299475Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"test-preemption-trainjob-gsqlf","namespace":"test-ns-xxbws"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"Started","action":"Started","note":"Admitted by clusterQueue cq-4rjwk"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:51.804037163Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"fe89789c-6b96-4af7-a40c-0d44ddb44124","job":"test-ns-xxbws/test-preemption-trainjob-gsqlf","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-3)","ts":"2026-11-02T22:13:51.858073948Z","logger":"trainjob","caller":"jobframework/reconciler.go:594","msg":"Unsuspending job","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"fe89789c-6b96-4af7-a40c-0d44ddb44124","job":"test-ns-xxbws/test-preemption-trainjob-gsqlf","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-preemption-trainjob-gsqlf\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-3)","ts":"2026-11-02T22:13:51.858137278Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"fe89789c-6b96-4af7-a40c-0d44ddb44124","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-preemption-trainjob-gsqlf\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:51.858222432Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"df416d7f-7bad-48ce-b05d-8c5c227b5ece","job":"test-ns-xxbws/test-preemption-trainjob-gsqlf","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:52.014980491Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"87dd7885-5ddc-4b70-8ae5-0a1d959d1111","job":"test-ns-xxbws/test-preemption-trainjob-gsqlf","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:52.171100198Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"61b89c3e-427a-4d00-bafd-f7dcfab041d5","job":"test-ns-xxbws/test-preemption-trainjob-gsqlf","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:52.326585033Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"407ca947-0c92-43fb-b066-007dd948901e","job":"test-ns-xxbws/test-preemption-trainjob-gsqlf","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:52.482633733Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"323998d6-4b5c-41d9-938d-5d08b9c78dc2","job":"test-ns-xxbws/test-preemption-trainjob-gsqlf","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:52.639486892Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"b1f8df14-26cd-46db-acd8-ffdccde9117b","job":"test-ns-xxbws/test-preemption-trainjob-gsqlf","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:53.36863937Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-xxbws","name":"trainjob-test-preemption-trainjob-gsqlf-ffca7","reconcileID":"f1f414c2-6874-44b6-97e9-492da525d466"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:53.436646693Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"b4d84e8f-7679-4da2-870e-bb507834a38a","job":"test-ns-xxbws/test-preemption-trainjob-gsqlf","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:53.594043918Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"5ba6b984-8242-4237-9640-672d54d3ad02","job":"test-ns-xxbws/test-preemption-trainjob-gsqlf","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:54.359191864Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-gsqlf-ffca7","namespace":"test-ns-xxbws"},"queue":"lq-b4tfb","status":"admitted","clusterQueue":"cq-4rjwk"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:54.359218286Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-xxbws","name":"trainjob-test-preemption-trainjob-gsqlf-ffca7","reconcileID":"8428bbed-6d11-45d4-ac33-fe2e4eff1081"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:54.359254836Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-xxbws","name":"trainjob-test-preemption-trainjob-gsqlf-ffca7","reconcileID":"8428bbed-6d11-45d4-ac33-fe2e4eff1081","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T22:13:54.359262535Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1381","msg":"Workload will not be queued because the workload is not active","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-gsqlf-ffca7","namespace":"test-ns-xxbws"},"queue":"lq-b4tfb","status":"admitted","clusterQueue":"cq-4rjwk"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:54.359364692Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"b5cb2b27-ee51-4747-be41-d7c42f96ed20","job":"test-ns-xxbws/test-preemption-trainjob-gsqlf","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:54.359371813Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-xxbws","name":"trainjob-test-preemption-trainjob-gsqlf-ffca7","reconcileID":"5fa378c5-777f-4735-a1fc-f2762e590618"}
{"level":"debug","ts":"2026-11-02T22:13:54.469328542Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-test-preemption-trainjob-gsqlf-ffca7","namespace":"test-ns-xxbws"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"EvictedDueToDeactivated","action":"EvictedDueToDeactivated","note":"The workload is deactivated"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:54.469323518Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-gsqlf-ffca7","namespace":"test-ns-xxbws"},"queue":"lq-b4tfb","status":"admitted","clusterQueue":"cq-4rjwk"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:54.469346022Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-xxbws","name":"trainjob-test-preemption-trainjob-gsqlf-ffca7","reconcileID":"3216e800-5839-4f81-af60-d2b3275f8c6f"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:54.469387026Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-xxbws","name":"trainjob-test-preemption-trainjob-gsqlf-ffca7","reconcileID":"3216e800-5839-4f81-af60-d2b3275f8c6f","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T22:13:54.469386449Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1381","msg":"Workload will not be queued because the workload is not active","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-gsqlf-ffca7","namespace":"test-ns-xxbws"},"queue":"lq-b4tfb","status":"admitted","clusterQueue":"cq-4rjwk"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:54.469488421Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-xxbws","name":"trainjob-test-preemption-trainjob-gsqlf-ffca7","reconcileID":"75f20a53-19cd-4f43-b0da-e531477c8996"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:54.515404277Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"0c7f39a1-04c6-430e-ac4f-25c26c3bec2a","job":"test-ns-xxbws/test-preemption-trainjob-gsqlf","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"info","ts":"2026-11-02T22:13:54.725360965Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-preemption-trainjob-gsqlf","namespace":"test-ns-xxbws"},"namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"51050ad6-e85f-44e1-9188-90f02b4e1438","replica-role":"leader"}
{"level":"error","ts":"2026-11-02T22:13:54.78072356Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"0c7f39a1-04c6-430e-ac4f-25c26c3bec2a","error":"jobs are still active","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"debug","ts":"2026-11-02T22:13:54.780734054Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"test-preemption-trainjob-gsqlf","namespace":"test-ns-xxbws"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"Stopped","action":"Stopped","note":"The workload is deactivated"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:54.780836074Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"f927d3c5-f338-41f4-9276-6dfcf47941a1","job":"test-ns-xxbws/test-preemption-trainjob-gsqlf","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-11-02T22:13:54.936255272Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"f927d3c5-f338-41f4-9276-6dfcf47941a1","error":"jobs are still active","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:54.936353836Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"c33936de-4f32-4be0-9528-f2338aafab8c","job":"test-ns-xxbws/test-preemption-trainjob-gsqlf","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-11-02T22:13:55.092429372Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"c33936de-4f32-4be0-9528-f2338aafab8c","error":"jobs are still active","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:55.092559784Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"3adc96c8-1e69-4ac6-8745-1e97ba41609e","job":"test-ns-xxbws/test-preemption-trainjob-gsqlf","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-11-02T22:13:55.24825648Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"3adc96c8-1e69-4ac6-8745-1e97ba41609e","error":"jobs are still active","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:55.248366884Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"3251b063-2c4c-4b60-b236-deaa4b1047b4","job":"test-ns-xxbws/test-preemption-trainjob-gsqlf","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:55.360060311Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-4rjwk","reconcileID":"c3a16f84-39b9-49e5-9391-1f1c4b8bb15b"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:55.36005839Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-xxbws","name":"lq-b4tfb","reconcileID":"c424f8e1-9db4-49c5-8be2-0b1ebbfb143b"}
{"level":"error","ts":"2026-11-02T22:13:55.403528811Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"3251b063-2c4c-4b60-b236-deaa4b1047b4","error":"jobs are still active","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:55.403630486Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"353d5353-29fd-4898-b712-aa6ad714a844","job":"test-ns-xxbws/test-preemption-trainjob-gsqlf","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:55.410365692Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-4rjwk"}}
{"level":"Level(-2)","ts":"2026-11-02T22:13:55.41038206Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-xxbws","name":"trainjob-test-preemption-trainjob-gsqlf-ffca7","reconcileID":"694a13d8-c228-4a29-8876-a664cb5e266c"}
{"level":"Level(-3)","ts":"2026-11-02T22:13:55.412541133Z","logger":"clusterqueue-reconciler","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","name":"cq-4rjwk","reconcileID":"c3a16f84-39b9-49e5-9391-1f1c4b8bb15b","error":"Operation cannot be fulfilled on clusterqueues.kueue.x-k8s.io \"cq-4rjwk\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:55.412603107Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-4rjwk","reconcileID":"18d96442-5154-4ba3-8fb2-0c0bc9dc86b8"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:55.414500735Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-b4tfb","namespace":"test-ns-xxbws"}}
{"level":"Level(-2)","ts":"2026-11-02T22:13:55.414585857Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-xxbws","name":"lq-b4tfb","reconcileID":"16bbcb9a-6166-43d1-8913-323cea6483e8"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:55.424396852Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:171","msg":"ResourceFlavor update event","replica-role":"leader","resourceFlavor":{"name":"rf-bkvnn"}}
{"level":"Level(-2)","ts":"2026-11-02T22:13:55.424445521Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-bkvnn","reconcileID":"28a7db60-53c7-416f-be8c-21ef5bb65f21"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:55.519350459Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:376","msg":"ClusterQueue delete event","replica-role":"leader","clusterQueue":{"name":"cq-4rjwk"}}
{"level":"Level(-2)","ts":"2026-11-02T22:13:55.519434086Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:385","msg":"Cleared resource metrics for deleted ClusterQueue.","replica-role":"leader","clusterQueue":{"name":"cq-4rjwk"}}
{"level":"Level(-2)","ts":"2026-11-02T22:13:55.519422894Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-xxbws","name":"trainjob-test-preemption-trainjob-gsqlf-ffca7","reconcileID":"63b6102d-d7db-40e3-8725-af79fa7ffb03"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:55.519442229Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-xxbws","name":"lq-b4tfb","reconcileID":"4d973083-4d57-4001-af15-3f38c5d90e79"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:55.519550978Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-bkvnn","reconcileID":"801cbfb1-ebb4-4a10-a65f-a5c6b194a8b9"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:55.575521576Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-b4tfb","namespace":"test-ns-xxbws"}}
{"level":"Level(-2)","ts":"2026-11-02T22:13:55.575611211Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-xxbws","name":"lq-b4tfb","reconcileID":"3702a754-7211-4991-823b-777696d6d447"}
{"level":"info","ts":"2026-11-02T22:13:55.613546048Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-preemption-trainjob-gsqlf","namespace":"test-ns-xxbws"},"namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"959ea2bf-3561-4f41-9691-527cf6c2c783","replica-role":"leader"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:55.624126785Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:159","msg":"ResourceFlavor delete event","replica-role":"leader","resourceFlavor":{"name":"rf-bkvnn"}}
{"level":"Level(-2)","ts":"2026-11-02T22:13:55.701023025Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:141","msg":"ResourceFlavor create event","replica-role":"leader","resourceFlavor":{"name":"rf-vj7rd"}}
{"level":"Level(-2)","ts":"2026-11-02T22:13:55.701083588Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-vj7rd","reconcileID":"04705f4e-25f6-4ebc-b7a1-ac6ef890197f"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:55.78808571Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"0d98f2d7-15b0-4751-afe1-ba034cf08e73","job":"test-ns-xxbws/test-preemption-trainjob-gsqlf","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:55.788535644Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-gsqlf-ffca7","namespace":"test-ns-xxbws"},"queue":"lq-b4tfb","status":"pending","prevStatus":"admitted","prevClusterQueue":"cq-4rjwk"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:55.788551671Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-xxbws","name":"trainjob-test-preemption-trainjob-gsqlf-ffca7","reconcileID":"ae2c985f-d2a9-4c16-9ef9-09da475a8d4d"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:55.7885813Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-xxbws","name":"trainjob-test-preemption-trainjob-gsqlf-ffca7","reconcileID":"ae2c985f-d2a9-4c16-9ef9-09da475a8d4d","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T22:13:55.788590832Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1381","msg":"Workload will not be queued because the workload is not active","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-gsqlf-ffca7","namespace":"test-ns-xxbws"},"queue":"lq-b4tfb","status":"pending","prevStatus":"admitted","prevClusterQueue":"cq-4rjwk"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:55.788672649Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-xxbws","name":"trainjob-test-preemption-trainjob-gsqlf-ffca7","reconcileID":"67d9fbb1-9da8-41ed-a575-3d0ef091816b"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:55.817665995Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:351","msg":"ClusterQueue create event","replica-role":"leader","clusterQueue":{"name":"cq-vmlds"}}
{"level":"Level(-2)","ts":"2026-11-02T22:13:55.817844871Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-vmlds","reconcileID":"414e353f-280c-479b-bde6-d9a330032e25"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:55.831197872Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:240","msg":"LocalQueue create event","replica-role":"leader","localQueue":{"name":"lq-kjl47","namespace":"test-ns-t762k"}}
{"level":"Level(-2)","ts":"2026-11-02T22:13:55.831297024Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-t762k","name":"lq-kjl47","reconcileID":"4986088a-1149-4428-92b8-3b30e1024064"}
{"level":"error","ts":"2026-11-02T22:13:55.839143074Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"0d98f2d7-15b0-4751-afe1-ba034cf08e73","job":"test-ns-xxbws/test-preemption-trainjob-gsqlf","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-gsqlf\" is forbidden: unable to create new content in namespace test-ns-xxbws because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-02T22:13:55.839192659Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"0d98f2d7-15b0-4751-afe1-ba034cf08e73","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-gsqlf\" is forbidden: unable to create new content in namespace test-ns-xxbws because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:55.839260511Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"fa5a30e4-c352-4ea0-8c1c-a7204cf639c2","job":"test-ns-xxbws/test-preemption-trainjob-gsqlf","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:55.874192564Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-vmlds"}}
{"level":"Level(-2)","ts":"2026-11-02T22:13:55.874386454Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-vmlds","reconcileID":"1631e0c4-4eff-4b71-8824-68ae5ee6ad77"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:55.884912333Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-t762k","name":"lq-kjl47","reconcileID":"d7f889a2-b28d-4f66-807a-d9fab60be19a"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:55.884935195Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-kjl47","namespace":"test-ns-t762k"}}
{"level":"error","ts":"2026-11-02T22:13:55.890394402Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"fa5a30e4-c352-4ea0-8c1c-a7204cf639c2","job":"test-ns-xxbws/test-preemption-trainjob-gsqlf","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-gsqlf\" is forbidden: unable to create new content in namespace test-ns-xxbws because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-02T22:13:55.890455937Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"fa5a30e4-c352-4ea0-8c1c-a7204cf639c2","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-gsqlf\" is forbidden: unable to create new content in namespace test-ns-xxbws because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-02T22:13:55.894630179Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:117","msg":"Validating create","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-runtimepatches-trainjob-hk4kj","namespace":"test-ns-t762k"},"namespace":"test-ns-t762k","name":"test-runtimepatches-trainjob-hk4kj","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:admin","requestID":"153139ad-0833-4bf3-82d0-0cd85f20b785","replica-role":"leader"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:55.898208895Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-gsqlf-ffca7","namespace":"test-ns-xxbws"},"queue":"lq-b4tfb","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:55.898246972Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1381","msg":"Workload will not be queued because the workload is not active","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-gsqlf-ffca7","namespace":"test-ns-xxbws"},"queue":"lq-b4tfb","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:55.898282618Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-xxbws","name":"trainjob-test-preemption-trainjob-gsqlf-ffca7","reconcileID":"2ed855bd-67dd-4e1a-bc77-7913ed37a8ac"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:55.898299827Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-xxbws","name":"trainjob-test-preemption-trainjob-gsqlf-ffca7","reconcileID":"f95b402d-c81c-492e-91a3-77579f0e4b59"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:55.898323329Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-xxbws","name":"trainjob-test-preemption-trainjob-gsqlf-ffca7","reconcileID":"2ed855bd-67dd-4e1a-bc77-7913ed37a8ac","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T22:13:55.898341509Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"ff7fb0e9-b0b1-40b6-9b32-82f7d9b77032","job":"test-ns-xxbws/test-preemption-trainjob-gsqlf","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:55.941444312Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-kjl47","namespace":"test-ns-t762k"}}
{"level":"Level(-2)","ts":"2026-11-02T22:13:55.941543315Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-t762k","name":"lq-kjl47","reconcileID":"1c4d225b-0535-4a6d-a343-d88ca2d7ec0c"}
{"level":"error","ts":"2026-11-02T22:13:55.949249296Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"ff7fb0e9-b0b1-40b6-9b32-82f7d9b77032","job":"test-ns-xxbws/test-preemption-trainjob-gsqlf","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-gsqlf\" is forbidden: unable to create new content in namespace test-ns-xxbws because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-02T22:13:55.949307425Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"ff7fb0e9-b0b1-40b6-9b32-82f7d9b77032","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-gsqlf\" is forbidden: unable to create new content in namespace test-ns-xxbws because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:55.949512489Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-t762k","name":"test-runtimepatches-trainjob-hk4kj","reconcileID":"c20ed330-acff-4143-8411-c317aee91ff7","job":"test-ns-t762k/test-runtimepatches-trainjob-hk4kj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:56.267430349Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1276","msg":"Workload create event","replica-role":"leader","workload":{"name":"trainjob-test-runtimepatches-trainjob-hk4kj-15d8b","namespace":"test-ns-t762k"},"queue":"lq-kjl47","status":"pending"}
{"level":"debug","ts":"2026-11-02T22:13:56.267455949Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"test-runtimepatches-trainjob-hk4kj","namespace":"test-ns-t762k"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"CreatedWorkload","action":"CreatedWorkload","note":"Created Workload: test-ns-t762k/trainjob-test-runtimepatches-trainjob-hk4kj-15d8b"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:56.267472822Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-t762k","name":"trainjob-test-runtimepatches-trainjob-hk4kj-15d8b","reconcileID":"67cc0efa-74ea-4f60-8f46-f8bd0da50bce"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:56.267518903Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-t762k","name":"trainjob-test-runtimepatches-trainjob-hk4kj-15d8b","reconcileID":"67cc0efa-74ea-4f60-8f46-f8bd0da50bce","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T22:13:56.267514473Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"17b3207a-fbc8-4144-9d33-7fb736cd7e83","job":"test-ns-xxbws/test-preemption-trainjob-gsqlf","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:56.267616068Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-t762k","name":"trainjob-test-runtimepatches-trainjob-hk4kj-15d8b","reconcileID":"5c060465-5d63-4934-bba4-64e34f3d31f0"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:56.267654686Z","logger":"scheduler","caller":"scheduler/scheduler.go:308","msg":"Obtained heads","replica-role":"leader","schedulingCycle":4,"headCount":1,"waitDuration":5.778969647}
{"level":"Level(-2)","ts":"2026-11-02T22:13:56.267697642Z","logger":"scheduler","caller":"scheduler/scheduler.go:323","msg":"Snapshot taken","replica-role":"leader","schedulingCycle":4,"duration":0.000014776}
{"level":"Level(-2)","ts":"2026-11-02T22:13:56.267792728Z","logger":"scheduler","caller":"scheduler/scheduler.go:328","msg":"Nomination done","replica-role":"leader","schedulingCycle":4,"entries":1,"inadmissibleEntries":0,"duration":0.000077666}
{"level":"Level(-2)","ts":"2026-11-02T22:13:56.267816271Z","logger":"scheduler","caller":"scheduler/scheduler.go:406","msg":"Attempting to schedule workload","replica-role":"leader","schedulingCycle":4,"workload":{"name":"trainjob-test-runtimepatches-trainjob-hk4kj-15d8b","namespace":"test-ns-t762k"},"clusterQueue":{"name":"cq-vmlds"}}
{"level":"Level(-2)","ts":"2026-11-02T22:13:56.267913246Z","logger":"scheduler","caller":"scheduler/scheduler.go:1041","msg":"Workload assumed in the cache","replica-role":"leader","schedulingCycle":4,"workload":{"name":"trainjob-test-runtimepatches-trainjob-hk4kj-15d8b","namespace":"test-ns-t762k"},"clusterQueue":{"name":"cq-vmlds"}}
{"level":"Level(-2)","ts":"2026-11-02T22:13:56.26793772Z","logger":"scheduler","caller":"scheduler/scheduler.go:362","msg":"Workload processing done","replica-role":"leader","schedulingCycle":4,"duration":0.000125518}
{"level":"Level(-2)","ts":"2026-11-02T22:13:56.267948908Z","logger":"scheduler","caller":"scheduler/scheduler.go:297","msg":"Scheduling cycle complete","replica-role":"leader","schedulingCycle":4,"duration":5.779264767}
{"level":"Level(-2)","ts":"2026-11-02T22:13:56.267988962Z","logger":"scheduler","caller":"scheduler/scheduler.go:295","msg":"Scheduling cycle starts","replica-role":"leader","schedulingCycle":5}
{"level":"error","ts":"2026-11-02T22:13:56.318754245Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"17b3207a-fbc8-4144-9d33-7fb736cd7e83","job":"test-ns-xxbws/test-preemption-trainjob-gsqlf","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-gsqlf\" is forbidden: unable to create new content in namespace test-ns-xxbws because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-02T22:13:56.318811892Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"17b3207a-fbc8-4144-9d33-7fb736cd7e83","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-gsqlf\" is forbidden: unable to create new content in namespace test-ns-xxbws because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:56.318893401Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-t762k","name":"test-runtimepatches-trainjob-hk4kj","reconcileID":"199f2b07-3b26-49c2-9ac9-3f4757f246d0","job":"test-ns-t762k/test-runtimepatches-trainjob-hk4kj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:56.37833584Z","logger":"scheduler","caller":"scheduler/scheduler.go:1000","msg":"Workload successfully admitted and assigned flavors","replica-role":"leader","schedulingCycle":4,"workload":{"name":"trainjob-test-runtimepatches-trainjob-hk4kj-15d8b","namespace":"test-ns-t762k"},"clusterQueue":{"name":"cq-vmlds"},"assignments":[{"name":"node","flavors":{"cpu":"rf-vj7rd","memory":"rf-vj7rd"},"resourceUsage":{"cpu":"100m","memory":"128Mi"},"count":1}]}
{"level":"debug","ts":"2026-11-02T22:13:56.378373659Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-test-runtimepatches-trainjob-hk4kj-15d8b","namespace":"test-ns-t762k"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"QuotaReserved","action":"QuotaReserved","note":"Quota reserved in ClusterQueue cq-vmlds, wait time since queued was 0s"}
{"level":"debug","ts":"2026-11-02T22:13:56.378404824Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-test-runtimepatches-trainjob-hk4kj-15d8b","namespace":"test-ns-t762k"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"Admitted","action":"Admitted","note":"Admitted by ClusterQueue cq-vmlds, wait time since reservation was 0s"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:56.378402787Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-runtimepatches-trainjob-hk4kj-15d8b","namespace":"test-ns-t762k"},"queue":"lq-kjl47","status":"admitted","prevStatus":"pending","clusterQueue":"cq-vmlds"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:56.378420558Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-t762k","name":"trainjob-test-runtimepatches-trainjob-hk4kj-15d8b","reconcileID":"f7a19463-93b0-4543-9e46-51dd6228f8b1"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:56.3784561Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-t762k","name":"trainjob-test-runtimepatches-trainjob-hk4kj-15d8b","reconcileID":"f7a19463-93b0-4543-9e46-51dd6228f8b1","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T22:13:56.37858977Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-t762k","name":"trainjob-test-runtimepatches-trainjob-hk4kj-15d8b","reconcileID":"acd5ae74-c3da-4d9a-b4b3-b8fd175a2f3b"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:56.475059086Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"4aff1e1f-12e5-4c70-a574-89b5151b0d86","job":"test-ns-xxbws/test-preemption-trainjob-gsqlf","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-11-02T22:13:56.526118881Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"4aff1e1f-12e5-4c70-a574-89b5151b0d86","job":"test-ns-xxbws/test-preemption-trainjob-gsqlf","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-gsqlf\" is forbidden: unable to create new content in namespace test-ns-xxbws because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-02T22:13:56.526172875Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"4aff1e1f-12e5-4c70-a574-89b5151b0d86","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-gsqlf\" is forbidden: unable to create new content in namespace test-ns-xxbws because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:56.526468366Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-t762k","name":"test-runtimepatches-trainjob-hk4kj","reconcileID":"5ccaeb17-fb72-4641-86ae-ffa9aa1cc628","job":"test-ns-t762k/test-runtimepatches-trainjob-hk4kj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:56.684530207Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-t762k","name":"test-runtimepatches-trainjob-hk4kj","reconcileID":"5ccaeb17-fb72-4641-86ae-ffa9aa1cc628","job":"test-ns-t762k/test-runtimepatches-trainjob-hk4kj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-3)","ts":"2026-11-02T22:13:56.738840524Z","logger":"trainjob","caller":"jobframework/reconciler.go:594","msg":"Unsuspending job","replica-role":"leader","namespace":"test-ns-t762k","name":"test-runtimepatches-trainjob-hk4kj","reconcileID":"5ccaeb17-fb72-4641-86ae-ffa9aa1cc628","job":"test-ns-t762k/test-runtimepatches-trainjob-hk4kj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-runtimepatches-trainjob-hk4kj\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-3)","ts":"2026-11-02T22:13:56.738907555Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-t762k","name":"test-runtimepatches-trainjob-hk4kj","reconcileID":"5ccaeb17-fb72-4641-86ae-ffa9aa1cc628","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-runtimepatches-trainjob-hk4kj\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:56.739008397Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-t762k","name":"test-runtimepatches-trainjob-hk4kj","reconcileID":"d3fcf369-b8e4-4520-8a4a-f85e2f2daaaf","job":"test-ns-t762k/test-runtimepatches-trainjob-hk4kj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:56.788928987Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-xxbws","name":"lq-b4tfb","reconcileID":"8bb6458f-b623-490c-a592-051df67db78d"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:56.896028945Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-t762k","name":"test-runtimepatches-trainjob-hk4kj","reconcileID":"d3fcf369-b8e4-4520-8a4a-f85e2f2daaaf","job":"test-ns-t762k/test-runtimepatches-trainjob-hk4kj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"info","ts":"2026-11-02T22:13:56.950344865Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-runtimepatches-trainjob-hk4kj","namespace":"test-ns-t762k"},"namespace":"test-ns-t762k","name":"test-runtimepatches-trainjob-hk4kj","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"598412f6-9e97-43ae-9733-8b60e4f13455","replica-role":"leader"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:57.010510352Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-vmlds"}}
{"level":"Level(-2)","ts":"2026-11-02T22:13:57.010611788Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-vmlds","reconcileID":"1b9e3b5b-551e-4dc4-aca7-6e65eb033fb1"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:57.010623495Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-t762k","name":"trainjob-test-runtimepatches-trainjob-hk4kj-15d8b","reconcileID":"c9873892-2e29-4a80-be4a-2962c3c04832"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:57.025587867Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:171","msg":"ResourceFlavor update event","replica-role":"leader","resourceFlavor":{"name":"rf-vj7rd"}}
{"level":"Level(-2)","ts":"2026-11-02T22:13:57.025667449Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-vj7rd","reconcileID":"3dc26864-b640-481f-bb96-c124953410ac"}
{"level":"info","ts":"2026-11-02T22:13:57.065333991Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-runtimepatches-trainjob-hk4kj","namespace":"test-ns-t762k"},"namespace":"test-ns-t762k","name":"test-runtimepatches-trainjob-hk4kj","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"0c318ba3-387e-40fc-888c-54667a6cfe46","replica-role":"leader"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:57.067994843Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-vmlds"}}
{"level":"Level(-2)","ts":"2026-11-02T22:13:57.068060909Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-vmlds","reconcileID":"d777a9d9-b117-4ec6-aaeb-f795d740d719"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:57.068109781Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-t762k","name":"lq-kjl47","reconcileID":"17cfa98d-7acd-4291-89ef-3a0c3976d968"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:57.068139226Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-t762k","name":"trainjob-test-runtimepatches-trainjob-hk4kj-15d8b","reconcileID":"ad5d23a0-6b65-487c-9eeb-d90796af9b2f"}
{"level":"debug","ts":"2026-11-02T22:13:57.123862646Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"test-runtimepatches-trainjob-hk4kj","namespace":"test-ns-t762k"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"Started","action":"Started","note":"Admitted by clusterQueue cq-vmlds"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:57.123903903Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"53ac161b-8b18-4b77-86a2-b54cfcea3a89","job":"test-ns-xxbws/test-preemption-trainjob-gsqlf","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:57.124554934Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-kjl47","namespace":"test-ns-t762k"}}
{"level":"Level(-2)","ts":"2026-11-02T22:13:57.124645205Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-t762k","name":"lq-kjl47","reconcileID":"cab377db-dc5d-4089-ab2f-d1ded0724ca2"}
{"level":"error","ts":"2026-11-02T22:13:57.174581953Z","logger":"events","caller":"events/event_broadcaster.go:270","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-t762k","name":"test-runtimepatches-trainjob-hk4kj.18e45a1e256e8026"},"error":"events.events.k8s.io \"test-runtimepatches-trainjob-hk4kj.18e45a1e256e8026\" is forbidden: unable to create new content in namespace test-ns-t762k because it is being terminated","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/workspace/upstream/kueue/src/vendor/k8s.io/client-go/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).attemptRecording\n\t/workspace/upstream/kueue/src/vendor/k8s.io/client-go/tools/events/event_broadcaster.go:211\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).recordToSink.func1\n\t/workspace/upstream/kueue/src/vendor/k8s.io/client-go/tools/events/event_broadcaster.go:200"}
{"level":"error","ts":"2026-11-02T22:13:57.175167661Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"53ac161b-8b18-4b77-86a2-b54cfcea3a89","job":"test-ns-xxbws/test-preemption-trainjob-gsqlf","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-gsqlf\" is forbidden: unable to create new content in namespace test-ns-xxbws because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-02T22:13:57.175227428Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"53ac161b-8b18-4b77-86a2-b54cfcea3a89","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-gsqlf\" is forbidden: unable to create new content in namespace test-ns-xxbws because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:57.175303985Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-t762k","name":"test-runtimepatches-trainjob-hk4kj","reconcileID":"3a158390-8983-40ec-9ddd-6f9f96dfaec0","job":"test-ns-t762k/test-runtimepatches-trainjob-hk4kj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-11-02T22:13:57.226753664Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-t762k","name":"test-runtimepatches-trainjob-hk4kj","reconcileID":"3a158390-8983-40ec-9ddd-6f9f96dfaec0","job":"test-ns-t762k/test-runtimepatches-trainjob-hk4kj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-runtimepatches-trainjob-hk4kj\" is forbidden: unable to create new content in namespace test-ns-t762k because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-02T22:13:57.226808746Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-t762k","name":"test-runtimepatches-trainjob-hk4kj","reconcileID":"3a158390-8983-40ec-9ddd-6f9f96dfaec0","error":"workloads.kueue.x-k8s.io \"test-runtimepatches-trainjob-hk4kj\" is forbidden: unable to create new content in namespace test-ns-t762k because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:57.232053908Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-t762k","name":"test-runtimepatches-trainjob-hk4kj","reconcileID":"1193c188-2d88-4127-8201-d3d594d9a219","job":"test-ns-t762k/test-runtimepatches-trainjob-hk4kj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-11-02T22:13:57.283623996Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-t762k","name":"test-runtimepatches-trainjob-hk4kj","reconcileID":"1193c188-2d88-4127-8201-d3d594d9a219","job":"test-ns-t762k/test-runtimepatches-trainjob-hk4kj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-runtimepatches-trainjob-hk4kj\" is forbidden: unable to create new content in namespace test-ns-t762k because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-02T22:13:57.283677458Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-t762k","name":"test-runtimepatches-trainjob-hk4kj","reconcileID":"1193c188-2d88-4127-8201-d3d594d9a219","error":"workloads.kueue.x-k8s.io \"test-runtimepatches-trainjob-hk4kj\" is forbidden: unable to create new content in namespace test-ns-t762k because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:57.293993059Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-t762k","name":"test-runtimepatches-trainjob-hk4kj","reconcileID":"d6de1c72-c246-47ba-9d88-4b588f71331a","job":"test-ns-t762k/test-runtimepatches-trainjob-hk4kj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-11-02T22:13:57.345587544Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-t762k","name":"test-runtimepatches-trainjob-hk4kj","reconcileID":"d6de1c72-c246-47ba-9d88-4b588f71331a","job":"test-ns-t762k/test-runtimepatches-trainjob-hk4kj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-runtimepatches-trainjob-hk4kj\" is forbidden: unable to create new content in namespace test-ns-t762k because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-02T22:13:57.345648018Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-t762k","name":"test-runtimepatches-trainjob-hk4kj","reconcileID":"d6de1c72-c246-47ba-9d88-4b588f71331a","error":"workloads.kueue.x-k8s.io \"test-runtimepatches-trainjob-hk4kj\" is forbidden: unable to create new content in namespace test-ns-t762k because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:57.345740966Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"f232b2fa-bf9b-495d-997c-eb834e771a9c","job":"test-ns-xxbws/test-preemption-trainjob-gsqlf","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-11-02T22:13:57.39732588Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"f232b2fa-bf9b-495d-997c-eb834e771a9c","job":"test-ns-xxbws/test-preemption-trainjob-gsqlf","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-gsqlf\" is forbidden: unable to create new content in namespace test-ns-xxbws because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-02T22:13:57.397408932Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-xxbws","name":"test-preemption-trainjob-gsqlf","reconcileID":"f232b2fa-bf9b-495d-997c-eb834e771a9c","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-gsqlf\" is forbidden: unable to create new content in namespace test-ns-xxbws because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:57.397522061Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-t762k","name":"test-runtimepatches-trainjob-hk4kj","reconcileID":"81d9fce4-1bb5-4b0c-967d-60f255a84ccb","job":"test-ns-t762k/test-runtimepatches-trainjob-hk4kj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-11-02T22:13:57.449353711Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-t762k","name":"test-runtimepatches-trainjob-hk4kj","reconcileID":"81d9fce4-1bb5-4b0c-967d-60f255a84ccb","job":"test-ns-t762k/test-runtimepatches-trainjob-hk4kj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-runtimepatches-trainjob-hk4kj\" is forbidden: unable to create new content in namespace test-ns-t762k because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-02T22:13:57.449411817Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-t762k","name":"test-runtimepatches-trainjob-hk4kj","reconcileID":"81d9fce4-1bb5-4b0c-967d-60f255a84ccb","error":"workloads.kueue.x-k8s.io \"test-runtimepatches-trainjob-hk4kj\" is forbidden: unable to create new content in namespace test-ns-t762k because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:57.455861413Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-t762k","name":"test-runtimepatches-trainjob-hk4kj","reconcileID":"f360fdd7-389a-48fe-8daa-6ebd8c50ee2d","job":"test-ns-t762k/test-runtimepatches-trainjob-hk4kj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}