{"level":"Level(-2)","ts":"2026-11-02T22:04:07.809380731Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-disabled","reconcileID":"8e24f794-a294-4004-91dc-19383268c833","job":"rhai-e2e-progression-qgx6z/progression-disabled","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:04:07.86770247Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-disabled","reconcileID":"bd9e5ed5-55ed-4712-a48d-a4d03d7ef628","job":"rhai-e2e-progression-qgx6z/progression-disabled","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:04:07.971402248Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-disabled","reconcileID":"b3159ca9-9abc-451f-a66e-bf5d138239b6","job":"rhai-e2e-progression-qgx6z/progression-disabled","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:04:15.06525211Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-invalid","reconcileID":"edcaf79e-ad44-4f66-b7df-a71b71cc049c","job":"rhai-e2e-progression-qgx6z/progression-invalid","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:04:15.236614082Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-invalid","reconcileID":"734aad6e-aa8f-4db4-b938-8f92e7000768","job":"rhai-e2e-progression-qgx6z/progression-invalid","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:04:15.466773065Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-invalid","reconcileID":"1e3b2818-694b-42ee-813c-145e3e48cc2a","job":"rhai-e2e-progression-qgx6z/progression-invalid","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:04:15.568412959Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-invalid","reconcileID":"c56c8f83-f249-475d-b168-9d1b5066109a","job":"rhai-e2e-progression-qgx6z/progression-invalid","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:04:15.590622627Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-invalid","reconcileID":"ea1e3f9f-b352-4fc1-81b9-15997e341c4f","job":"rhai-e2e-progression-qgx6z/progression-invalid","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:04:15.672350365Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-invalid","reconcileID":"115623a9-0955-41c2-84ac-d0746e546efd","job":"rhai-e2e-progression-qgx6z/progression-invalid","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:04:19.846773909Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-invalid","reconcileID":"d9621182-c159-4ef4-a386-2042d3707d61","job":"rhai-e2e-progression-qgx6z/progression-invalid","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:04:19.950235286Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-invalid","reconcileID":"528e5186-ee26-4e1c-bae6-1f90a6732c71","job":"rhai-e2e-progression-qgx6z/progression-invalid","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:04:30.251029026Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-custom-config","reconcileID":"be9f0208-ff78-4e71-8af4-cda165e0423d","job":"rhai-e2e-progression-qgx6z/progression-custom-config","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:04:30.412864392Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-custom-config","reconcileID":"aac86160-fbbc-46a4-9381-8998d6436213","job":"rhai-e2e-progression-qgx6z/progression-custom-config","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:04:30.447463783Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-min-interval","reconcileID":"9a4d771c-c6e0-4133-aa20-32eabe3b1914","job":"rhai-e2e-progression-qgx6z/progression-min-interval","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:04:30.643252909Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-custom-config","reconcileID":"1f8140a9-6f95-4a08-8424-4b54ba3af93a","job":"rhai-e2e-progression-qgx6z/progression-custom-config","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:04:30.644654087Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-max-interval","reconcileID":"633dfccb-2242-4d47-874c-1a5a11d07c9f","job":"rhai-e2e-progression-qgx6z/progression-max-interval","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:04:30.761661151Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-custom-config","reconcileID":"b3b0a4e2-27ed-4869-9e64-444db1a9c7d3","job":"rhai-e2e-progression-qgx6z/progression-custom-config","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:04:30.840164486Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-default-interval","reconcileID":"8ca586fc-30bb-49d3-9dff-e9941586672d","job":"rhai-e2e-progression-qgx6z/progression-default-interval","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:04:31.1246849Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-job-failure","reconcileID":"3ffa5592-3519-4cb8-9495-73e50cef3c4d","job":"rhai-e2e-progression-qgx6z/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:04:32.321490767Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-custom-config","reconcileID":"dd6c5cdc-123c-4ce1-98c1-8ad4f54ca894","job":"rhai-e2e-progression-qgx6z/progression-custom-config","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:04:32.821988982Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-job-failure","reconcileID":"0be226f2-a129-4e38-9a6d-ad3f1d91d16c","job":"rhai-e2e-progression-qgx6z/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:04:33.052319417Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-job-failure","reconcileID":"d5fc9595-bf2a-4d33-97d1-7a4cb3e886bf","job":"rhai-e2e-progression-qgx6z/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:04:33.171145672Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-job-failure","reconcileID":"82806659-8735-4cf5-a8e8-2b08f4ffe694","job":"rhai-e2e-progression-qgx6z/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:04:34.932410645Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-job-failure","reconcileID":"a0ff1637-f042-45b3-b9d2-5cdbb390f473","job":"rhai-e2e-progression-qgx6z/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:04:40.936276011Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-invalid","reconcileID":"04705362-872e-444d-ad68-c0c6228d5845","job":"rhai-e2e-progression-qgx6z/progression-invalid","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:04:41.946860084Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-invalid","reconcileID":"72d2cca9-b487-40bc-9a1e-62182954e87f","job":"rhai-e2e-progression-qgx6z/progression-invalid","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:04:42.105594428Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-invalid","reconcileID":"0ce1944b-393b-4314-8db0-620d7fcc4f7e","job":"rhai-e2e-progression-qgx6z/progression-invalid","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:04:42.233079167Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-invalid","reconcileID":"fb43ea20-c838-4932-8791-9ee71c00033d","job":"rhai-e2e-progression-qgx6z/progression-invalid","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:10.244606532Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-custom-config","reconcileID":"fce5e305-2f25-4b26-868c-8304000ba9c8","job":"rhai-e2e-progression-qgx6z/progression-custom-config","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:10.299435226Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-custom-config","reconcileID":"1642dd6d-de7e-4402-92ef-53b4f57aa974","job":"rhai-e2e-progression-qgx6z/progression-custom-config","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:11.428227055Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-job-failure","reconcileID":"379ba38b-385a-4147-8eb8-d75655d7dbc1","job":"rhai-e2e-progression-qgx6z/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:11.531657963Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-job-failure","reconcileID":"f887247c-d300-42e6-98ff-41c0c8f8805a","job":"rhai-e2e-progression-qgx6z/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:11.813492035Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-job-failure","reconcileID":"6fcdec86-ce01-4a6d-b1f6-79489d019acc","job":"rhai-e2e-progression-qgx6z/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:12.022232755Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-job-failure","reconcileID":"491e0abc-1d75-40c8-a8b4-a83778760880","job":"rhai-e2e-progression-qgx6z/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:12.229168331Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-job-failure","reconcileID":"9d6e94fa-8a0f-4458-b2ee-82b540520378","job":"rhai-e2e-progression-qgx6z/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:12.437471109Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-job-failure","reconcileID":"d29ef185-2e39-4e85-b8a9-6425ef501983","job":"rhai-e2e-progression-qgx6z/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:12.851393012Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-job-failure","reconcileID":"0ff55980-d9d0-432c-82d5-15826a69f16a","job":"rhai-e2e-progression-qgx6z/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:18.470523973Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-job-failure","reconcileID":"d3d2a4ff-7f50-475b-9b20-52e7c53b868c","job":"rhai-e2e-progression-qgx6z/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:18.574859371Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-job-failure","reconcileID":"306791dd-5e94-4a49-99e3-5606db508db6","job":"rhai-e2e-progression-qgx6z/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:19.482726755Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-job-failure","reconcileID":"46ca7db0-bfcf-4558-b239-2e080a356b5d","job":"rhai-e2e-progression-qgx6z/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:19.583776996Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-job-failure","reconcileID":"a68c05f1-721d-44bb-8104-f128abe26c67","job":"rhai-e2e-progression-qgx6z/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:19.639981533Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-job-failure","reconcileID":"32a8d22e-21cd-4a4f-805f-ba64ed32ca0b","job":"rhai-e2e-progression-qgx6z/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:20.077779206Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-job-failure","reconcileID":"1e85e058-3a26-4b4c-9b15-9094747cf5ca","job":"rhai-e2e-progression-qgx6z/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:20.235690566Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-job-failure","reconcileID":"a378630b-e46f-402c-a128-f82150c37aa2","job":"rhai-e2e-progression-qgx6z/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:20.591347303Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-no-metrics","reconcileID":"223e3e99-0c60-4f57-9042-eebc6b481a80","job":"rhai-e2e-progression-qgx6z/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:21.046567413Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-no-metrics","reconcileID":"1f0e3401-0206-4c1f-96fe-003d2e3fea61","job":"rhai-e2e-progression-qgx6z/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:21.279528432Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-no-metrics","reconcileID":"6d3bb3ce-b26c-4377-889d-cb36352fbc8b","job":"rhai-e2e-progression-qgx6z/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:21.593441317Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-no-metrics","reconcileID":"91f8c106-0003-483a-bb68-291ad2660b9a","job":"rhai-e2e-progression-qgx6z/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:21.640907541Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-no-metrics","reconcileID":"9e1c3517-7aee-4fc5-8c24-199bc87016a1","job":"rhai-e2e-progression-qgx6z/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:22.083083913Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-no-metrics","reconcileID":"850685dc-0841-4703-ac72-707cb0e26230","job":"rhai-e2e-progression-qgx6z/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:23.106520786Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-job-failure","reconcileID":"15e68c56-26bb-4086-a41f-fd53e0eb787b","job":"rhai-e2e-progression-qgx6z/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:23.315238075Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-job-failure","reconcileID":"cdf592b6-2141-421c-8540-56d6050dd785","job":"rhai-e2e-progression-qgx6z/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:24.376264743Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-no-metrics","reconcileID":"6a8219e5-8134-4d89-b78b-6fc3ac2b35d8","job":"rhai-e2e-progression-qgx6z/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:24.477944223Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-no-metrics","reconcileID":"a18443f7-55a1-4f5f-b584-f6845b561f6d","job":"rhai-e2e-progression-qgx6z/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:29.39870904Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-no-metrics","reconcileID":"7722bb2a-acb9-472b-9cf0-07bd857c34e0","job":"rhai-e2e-progression-qgx6z/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:29.502565974Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-no-metrics","reconcileID":"b0b08c79-061e-4c8f-880c-dc214c5bb007","job":"rhai-e2e-progression-qgx6z/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:30.406704419Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-no-metrics","reconcileID":"28301531-8026-4fe5-a58d-94045b44b644","job":"rhai-e2e-progression-qgx6z/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:30.508800413Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-no-metrics","reconcileID":"97e21048-5f1c-4313-8ee3-785f2eeafd31","job":"rhai-e2e-progression-qgx6z/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:30.524496313Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-no-metrics","reconcileID":"d136d9cb-40de-4dfa-b2e5-7a851ccce2dc","job":"rhai-e2e-progression-qgx6z/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:31.003538403Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-no-metrics","reconcileID":"863c2d8b-e8ff-4be6-a7fa-c1dc2264f7c8","job":"rhai-e2e-progression-qgx6z/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:31.162335063Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-no-metrics","reconcileID":"819a4a0b-3f8a-4917-a5fa-c8d79feebbef","job":"rhai-e2e-progression-qgx6z/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:31.347509949Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-custom-config","reconcileID":"e1564b56-0e6d-4301-914d-eaa79a7a10dd","job":"rhai-e2e-progression-qgx6z/progression-custom-config","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:31.402425752Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-custom-config","reconcileID":"3fe2cc71-3bd1-4342-b894-42943f2b16c1","job":"rhai-e2e-progression-qgx6z/progression-custom-config","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:32.358048618Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-custom-config","reconcileID":"7dcddc14-3c6f-4cd4-9372-8724f83b01ac","job":"rhai-e2e-progression-qgx6z/progression-custom-config","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:32.410310824Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-custom-config","reconcileID":"a9f37758-4cf6-4c28-8ac2-aebc7c7cf4ba","job":"rhai-e2e-progression-qgx6z/progression-custom-config","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:32.459652042Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-custom-config","reconcileID":"7cc39b68-e1ba-442e-aff7-583414043384","job":"rhai-e2e-progression-qgx6z/progression-custom-config","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:06:32.806464615Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-qgx6z","name":"progression-custom-config","reconcileID":"f3bd29ee-5bf2-4eaa-87f1-08ec9701e41b","job":"rhai-e2e-progression-qgx6z/progression-custom-config","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-qgx6z"}
{"level":"Level(-2)","ts":"2026-11-02T22:07:27.10342062Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wc4lt","name":"test-trainjob-66k42","reconcileID":"58069f89-5c15-4a64-8830-7c0fe79f5276","job":"test-ns-wc4lt/test-trainjob-66k42","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wc4lt"}
{"level":"Level(-2)","ts":"2026-11-02T22:07:27.285051865Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wc4lt","name":"test-trainjob-66k42","reconcileID":"f355e0d7-76f8-4445-91cb-1c33a74cde5f","job":"test-ns-wc4lt/test-trainjob-66k42","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wc4lt"}
{"level":"Level(-2)","ts":"2026-11-02T22:07:27.514294474Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wc4lt","name":"test-trainjob-66k42","reconcileID":"779db120-6356-4b7b-90a8-7741c7ce0ea8","job":"test-ns-wc4lt/test-trainjob-66k42","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wc4lt"}
{"level":"Level(-2)","ts":"2026-11-02T22:07:27.615923297Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wc4lt","name":"test-trainjob-66k42","reconcileID":"27e3ad13-ade0-4672-b471-fd88c9dc4651","job":"test-ns-wc4lt/test-trainjob-66k42","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wc4lt"}
{"level":"Level(-2)","ts":"2026-11-02T22:07:27.626587553Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wc4lt","name":"test-trainjob-66k42","reconcileID":"a997ad31-8591-4cc1-ab4f-6abac3dec7a3","job":"test-ns-wc4lt/test-trainjob-66k42","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wc4lt"}
{"level":"Level(-2)","ts":"2026-11-02T22:07:27.719497149Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wc4lt","name":"test-trainjob-66k42","reconcileID":"58ea9460-b88b-4240-98fb-bc4614903347","job":"test-ns-wc4lt/test-trainjob-66k42","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wc4lt"}
{"level":"Level(-2)","ts":"2026-11-02T22:07:29.649242364Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wc4lt","name":"test-trainjob-66k42","reconcileID":"0dbdfe06-53f7-4302-9883-6e97a21b4ab5","job":"test-ns-wc4lt/test-trainjob-66k42","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wc4lt"}
{"level":"Level(-2)","ts":"2026-11-02T22:07:29.750804381Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wc4lt","name":"test-trainjob-66k42","reconcileID":"5bcf6ebc-35fc-493f-a10a-be436b355fdb","job":"test-ns-wc4lt/test-trainjob-66k42","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wc4lt"}
{"level":"Level(-2)","ts":"2026-11-02T22:07:34.67672997Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wc4lt","name":"test-trainjob-66k42","reconcileID":"5e0f8f27-cb1c-427d-b7a3-804bc2f6795d","job":"test-ns-wc4lt/test-trainjob-66k42","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wc4lt"}
{"level":"Level(-2)","ts":"2026-11-02T22:07:34.778472106Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wc4lt","name":"test-trainjob-66k42","reconcileID":"83edd09f-beb4-4a98-aafa-9cae0246c556","job":"test-ns-wc4lt/test-trainjob-66k42","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wc4lt"}
{"level":"Level(-2)","ts":"2026-11-02T22:07:35.687725198Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wc4lt","name":"test-trainjob-66k42","reconcileID":"04ec258d-42dd-4ac9-9450-4fa12ba118cb","job":"test-ns-wc4lt/test-trainjob-66k42","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wc4lt"}
{"level":"Level(-2)","ts":"2026-11-02T22:07:35.751900593Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wc4lt","name":"test-trainjob-66k42","reconcileID":"35f3b910-b8b7-4e59-bd63-cd566e3f39c9","job":"test-ns-wc4lt/test-trainjob-66k42","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wc4lt"}
{"level":"Level(-2)","ts":"2026-11-02T22:07:35.789811551Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wc4lt","name":"test-trainjob-66k42","reconcileID":"364dca2c-3bf8-4469-b114-525441221611","job":"test-ns-wc4lt/test-trainjob-66k42","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wc4lt"}
{"level":"Level(-2)","ts":"2026-11-02T22:07:35.891204601Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wc4lt","name":"test-trainjob-66k42","reconcileID":"75769c8c-8870-40d1-b03e-722c5915e4dd","job":"test-ns-wc4lt/test-trainjob-66k42","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wc4lt"}
{"level":"Level(-2)","ts":"2026-11-02T22:07:36.376785397Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jddbc","name":"test-trainjob-bdmwc","reconcileID":"87970739-6dc3-45a8-b43d-cb378cd72ace","job":"test-ns-jddbc/test-trainjob-bdmwc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jddbc"}
{"level":"Level(-2)","ts":"2026-11-02T22:07:36.545561169Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jddbc","name":"test-trainjob-bdmwc","reconcileID":"2af797c5-f72b-4364-b827-50eec5ead2d9","job":"test-ns-jddbc/test-trainjob-bdmwc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jddbc"}
{"level":"Level(-2)","ts":"2026-11-02T22:07:36.7758077Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jddbc","name":"test-trainjob-bdmwc","reconcileID":"9d80bcf5-444c-432f-ad0b-21797d54b2e4","job":"test-ns-jddbc/test-trainjob-bdmwc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jddbc"}
{"level":"Level(-2)","ts":"2026-11-02T22:07:36.878732822Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jddbc","name":"test-trainjob-bdmwc","reconcileID":"b513242b-2c28-4bbb-b4f3-7b8d582d185f","job":"test-ns-jddbc/test-trainjob-bdmwc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jddbc"}
{"level":"Level(-2)","ts":"2026-11-02T22:07:36.891335814Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jddbc","name":"test-trainjob-bdmwc","reconcileID":"d913772c-30c6-4bf3-abce-830b8a126d63","job":"test-ns-jddbc/test-trainjob-bdmwc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jddbc"}
{"level":"Level(-2)","ts":"2026-11-02T22:07:36.981579316Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jddbc","name":"test-trainjob-bdmwc","reconcileID":"46c5a7a6-2505-4458-83f0-b0552b5d6f5d","job":"test-ns-jddbc/test-trainjob-bdmwc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jddbc"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:24.391746322Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jddbc","name":"test-trainjob-bdmwc","reconcileID":"2948829e-39f0-438c-ae23-2be9e9d953f0","job":"test-ns-jddbc/test-trainjob-bdmwc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jddbc"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:24.493081564Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jddbc","name":"test-trainjob-bdmwc","reconcileID":"4a06abef-a359-4896-8b02-583e38776ad1","job":"test-ns-jddbc/test-trainjob-bdmwc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jddbc"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:28.347208239Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jddbc","name":"test-trainjob-bdmwc","reconcileID":"21c30369-d559-4061-ba37-b2696af75417","job":"test-ns-jddbc/test-trainjob-bdmwc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jddbc"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:28.453634247Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jddbc","name":"test-trainjob-bdmwc","reconcileID":"bc0f22c6-b7a3-4f32-95d3-8e47088b1f9d","job":"test-ns-jddbc/test-trainjob-bdmwc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jddbc"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:29.355517792Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jddbc","name":"test-trainjob-bdmwc","reconcileID":"360db7b4-43d1-451a-a496-0907956e9f4b","job":"test-ns-jddbc/test-trainjob-bdmwc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jddbc"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:29.419500384Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jddbc","name":"test-trainjob-bdmwc","reconcileID":"66be3e6e-f11c-4a00-9968-d5c47cae2832","job":"test-ns-jddbc/test-trainjob-bdmwc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jddbc"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:29.459510243Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jddbc","name":"test-trainjob-bdmwc","reconcileID":"3f0e198c-76d9-430d-8b06-3cc5d7d67097","job":"test-ns-jddbc/test-trainjob-bdmwc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jddbc"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:29.662787803Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jddbc","name":"test-trainjob-bdmwc","reconcileID":"d6469169-1bdd-4e62-afb4-1d3d639d2c60","job":"test-ns-jddbc/test-trainjob-bdmwc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jddbc"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:30.061764035Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-6d7nn","name":"test-trainjob-mp7h4","reconcileID":"8af8500a-7551-40ea-8b48-ce4f35d1b183","job":"test-ns-6d7nn/test-trainjob-mp7h4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-6d7nn"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:30.460558834Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-6d7nn","name":"test-trainjob-mp7h4","reconcileID":"4a62e2e5-795e-4a5d-95d7-4140ef9c8e03","job":"test-ns-6d7nn/test-trainjob-mp7h4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-6d7nn"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:30.690092303Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-6d7nn","name":"test-trainjob-mp7h4","reconcileID":"55590743-6aff-497a-bb5e-d18d046da833","job":"test-ns-6d7nn/test-trainjob-mp7h4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-6d7nn"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:30.79146694Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-6d7nn","name":"test-trainjob-mp7h4","reconcileID":"ac681795-d7ca-4350-a592-ec432368a5f0","job":"test-ns-6d7nn/test-trainjob-mp7h4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-6d7nn"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:31.004112044Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-6d7nn","name":"test-trainjob-mp7h4","reconcileID":"fad5dab4-0c4a-4e44-acd8-c7f84d9c0b4f","job":"test-ns-6d7nn/test-trainjob-mp7h4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-6d7nn"}
{"level":"Level(-2)","ts":"2026-11-02T22:13:31.107498885Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-6d7nn","name":"test-trainjob-mp7h4","reconcileID":"ea58a65c-4ea0-4cf9-8d78-9d2198e50665","job":"test-ns-6d7nn/test-trainjob-mp7h4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-6d7nn"}
{"level":"Level(-2)","ts":"2026-11-02T22:16:36.052825371Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-6d7nn","name":"test-trainjob-mp7h4","reconcileID":"f64731bb-8397-4307-9c81-a6f177255cef","job":"test-ns-6d7nn/test-trainjob-mp7h4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-6d7nn"}
{"level":"Level(-2)","ts":"2026-11-02T22:16:36.15489137Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-6d7nn","name":"test-trainjob-mp7h4","reconcileID":"551ae9a5-df01-4ceb-9bee-45875f4493a4","job":"test-ns-6d7nn/test-trainjob-mp7h4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-6d7nn"}
{"level":"Level(-2)","ts":"2026-11-02T22:16:38.063582999Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-6d7nn","name":"test-trainjob-mp7h4","reconcileID":"e8952d7d-8753-4324-9935-4906752e421c","job":"test-ns-6d7nn/test-trainjob-mp7h4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-6d7nn"}
{"level":"Level(-2)","ts":"2026-11-02T22:16:38.165028412Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-6d7nn","name":"test-trainjob-mp7h4","reconcileID":"748cec35-c34f-4dea-be0d-bd4d776129a3","job":"test-ns-6d7nn/test-trainjob-mp7h4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-6d7nn"}
{"level":"Level(-2)","ts":"2026-11-02T22:16:39.073620142Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-6d7nn","name":"test-trainjob-mp7h4","reconcileID":"2944a92c-3cb2-464f-88d8-018342c2eea0","job":"test-ns-6d7nn/test-trainjob-mp7h4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-6d7nn"}
{"level":"Level(-2)","ts":"2026-11-02T22:16:39.135676878Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-6d7nn","name":"test-trainjob-mp7h4","reconcileID":"1070a629-135e-4712-a54c-09c42ae83a27","job":"test-ns-6d7nn/test-trainjob-mp7h4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-6d7nn"}
{"level":"Level(-2)","ts":"2026-11-02T22:16:39.175993069Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-6d7nn","name":"test-trainjob-mp7h4","reconcileID":"3de0a0d8-ffa6-4d3a-8f20-caaec32ab063","job":"test-ns-6d7nn/test-trainjob-mp7h4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-6d7nn"}
{"level":"Level(-2)","ts":"2026-11-02T22:16:39.279357622Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-6d7nn","name":"test-trainjob-mp7h4","reconcileID":"a1036660-75e3-4ec0-a869-93a21ccbb15e","job":"test-ns-6d7nn/test-trainjob-mp7h4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-6d7nn"}
{"level":"Level(-2)","ts":"2026-11-02T22:16:40.807468258Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ptctd","name":"test-trainjob-vkxgp","reconcileID":"aef07a7a-bce1-415c-bafb-7583209bd4a0","job":"test-ns-ptctd/test-trainjob-vkxgp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ptctd"}
{"level":"Level(-2)","ts":"2026-11-02T22:16:41.216626224Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ptctd","name":"test-trainjob-vkxgp","reconcileID":"c4703fdf-7b94-4f7c-b4d9-4f364d06bb5d","job":"test-ns-ptctd/test-trainjob-vkxgp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ptctd"}
{"level":"Level(-2)","ts":"2026-11-02T22:16:41.452114071Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ptctd","name":"test-trainjob-vkxgp","reconcileID":"417d54fd-0c7b-48fe-8859-9df17cf89d3b","job":"test-ns-ptctd/test-trainjob-vkxgp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ptctd"}
{"level":"Level(-2)","ts":"2026-11-02T22:16:41.557435226Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ptctd","name":"test-trainjob-vkxgp","reconcileID":"4ff46700-703f-439a-ab0a-0151d62a927c","job":"test-ns-ptctd/test-trainjob-vkxgp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ptctd"}
{"level":"Level(-2)","ts":"2026-11-02T22:16:41.782793338Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ptctd","name":"test-trainjob-vkxgp","reconcileID":"c3a529ce-d4ba-4ec5-bf29-6b24f0511520","job":"test-ns-ptctd/test-trainjob-vkxgp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ptctd"}
{"level":"Level(-2)","ts":"2026-11-02T22:16:41.887412147Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ptctd","name":"test-trainjob-vkxgp","reconcileID":"c8b646bf-603f-4b29-bdb7-0b969bacfb99","job":"test-ns-ptctd/test-trainjob-vkxgp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ptctd"}
{"level":"Level(-2)","ts":"2026-11-02T22:16:45.488382964Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wc4lt","name":"test-trainjob-66k42","reconcileID":"ef7a3a96-bf09-4e78-8f3d-d46bda474fe6","job":"test-ns-wc4lt/test-trainjob-66k42","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wc4lt"}
{"level":"Level(-2)","ts":"2026-11-02T22:16:45.581805491Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jddbc","name":"test-trainjob-bdmwc","reconcileID":"e098425e-c402-41e2-9ab1-6a8e87cb740e","job":"test-ns-jddbc/test-trainjob-bdmwc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jddbc"}
{"level":"Level(-2)","ts":"2026-11-02T22:16:45.707650637Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jddbc","name":"test-trainjob-bdmwc","reconcileID":"0e674561-f1a5-4850-aed1-7a90f7bc5d03","job":"test-ns-jddbc/test-trainjob-bdmwc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jddbc"}
{"level":"Level(-2)","ts":"2026-11-02T22:16:46.31860334Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-6d7nn","name":"test-trainjob-mp7h4","reconcileID":"ee4a85fd-c307-4490-a1ed-fcb19ce0c52e","job":"test-ns-6d7nn/test-trainjob-mp7h4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-6d7nn"}
{"level":"Level(-2)","ts":"2026-11-02T22:16:46.434734953Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jddbc","name":"test-trainjob-bdmwc","reconcileID":"ff587601-b9ff-45a2-b1e5-ec0a7050863c","job":"test-ns-jddbc/test-trainjob-bdmwc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jddbc"}
{"level":"Level(-2)","ts":"2026-11-02T22:16:57.244237607Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ptctd","name":"test-trainjob-vkxgp","reconcileID":"ec2b552e-36eb-4a1c-835c-0134b39b1a49","job":"test-ns-ptctd/test-trainjob-vkxgp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ptctd"}
{"level":"Level(-2)","ts":"2026-11-02T22:16:57.304236635Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ptctd","name":"test-trainjob-vkxgp","reconcileID":"a69a15e9-fc04-49f1-b220-182e9c95d8ef","job":"test-ns-ptctd/test-trainjob-vkxgp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ptctd"}
{"level":"Level(-2)","ts":"2026-11-02T22:17:02.219035504Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ptctd","name":"test-trainjob-vkxgp","reconcileID":"95494a67-f991-47af-a4c6-44d3188b4a5a","job":"test-ns-ptctd/test-trainjob-vkxgp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ptctd"}
{"level":"Level(-2)","ts":"2026-11-02T22:17:02.324595489Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ptctd","name":"test-trainjob-vkxgp","reconcileID":"a03ff1d5-c722-4aa1-adbb-e8779bb50dcd","job":"test-ns-ptctd/test-trainjob-vkxgp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ptctd"}
{"level":"Level(-2)","ts":"2026-11-02T22:17:03.226704061Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ptctd","name":"test-trainjob-vkxgp","reconcileID":"2c647a86-7476-441e-a879-ce0e442edcdc","job":"test-ns-ptctd/test-trainjob-vkxgp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ptctd"}
{"level":"Level(-2)","ts":"2026-11-02T22:17:03.334012274Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ptctd","name":"test-trainjob-vkxgp","reconcileID":"960980f6-24fd-4061-a3d9-6ad95ae0af86","job":"test-ns-ptctd/test-trainjob-vkxgp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ptctd"}
{"level":"Level(-2)","ts":"2026-11-02T22:17:03.459781638Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ptctd","name":"test-trainjob-vkxgp","reconcileID":"d95ea2f3-7000-434f-840e-4cc9ed5c4be5","job":"test-ns-ptctd/test-trainjob-vkxgp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ptctd"}
{"level":"Level(-2)","ts":"2026-11-02T22:17:03.565388196Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ptctd","name":"test-trainjob-vkxgp","reconcileID":"f5e902fc-466e-4be9-9803-803e1ed1c3c5","job":"test-ns-ptctd/test-trainjob-vkxgp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ptctd"}
{"level":"Level(-2)","ts":"2026-11-02T22:17:03.586154525Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ptctd","name":"test-trainjob-vkxgp","reconcileID":"b99f150f-1cf1-4b2c-a811-35be6b608c3e","job":"test-ns-ptctd/test-trainjob-vkxgp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ptctd"}
{"level":"Level(-2)","ts":"2026-11-02T22:17:03.672237988Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ptctd","name":"test-trainjob-vkxgp","reconcileID":"d08523cc-f9ea-4f52-8e41-31b6f86055e7","job":"test-ns-ptctd/test-trainjob-vkxgp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ptctd"}
{"level":"Level(-2)","ts":"2026-11-02T22:17:54.46050742Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ptctd","name":"test-trainjob-vkxgp","reconcileID":"15d46127-45eb-4211-8291-645d8a377f6b","job":"test-ns-ptctd/test-trainjob-vkxgp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ptctd"}
{"level":"Level(-2)","ts":"2026-11-02T22:17:54.566106077Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ptctd","name":"test-trainjob-vkxgp","reconcileID":"a645e71c-3d05-4f57-879b-6f10343f6dc8","job":"test-ns-ptctd/test-trainjob-vkxgp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ptctd"}
{"level":"Level(-2)","ts":"2026-11-02T22:18:05.506971461Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ptctd","name":"test-trainjob-vkxgp","reconcileID":"4afd5be9-106a-45fe-962f-2f085c1d9575","job":"test-ns-ptctd/test-trainjob-vkxgp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ptctd"}
{"level":"Level(-2)","ts":"2026-11-02T22:18:05.612807983Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ptctd","name":"test-trainjob-vkxgp","reconcileID":"fcc4bf02-41e1-49fc-84f6-6a8835a0c12a","job":"test-ns-ptctd/test-trainjob-vkxgp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ptctd"}
{"level":"Level(-2)","ts":"2026-11-02T22:18:06.516601464Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ptctd","name":"test-trainjob-vkxgp","reconcileID":"f5b5bdb9-6b21-4664-98b9-00a60f229ac8","job":"test-ns-ptctd/test-trainjob-vkxgp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ptctd"}
{"level":"Level(-2)","ts":"2026-11-02T22:18:06.622639915Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ptctd","name":"test-trainjob-vkxgp","reconcileID":"edd423b0-b121-4536-b899-ee3ce7be1c53","job":"test-ns-ptctd/test-trainjob-vkxgp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ptctd"}
{"level":"Level(-2)","ts":"2026-11-02T22:18:06.710703677Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ptctd","name":"test-trainjob-vkxgp","reconcileID":"7df9c08d-102e-4635-b37c-dec71bcb0c8e","job":"test-ns-ptctd/test-trainjob-vkxgp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ptctd"}
{"level":"Level(-2)","ts":"2026-11-02T22:18:06.818153275Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ptctd","name":"test-trainjob-vkxgp","reconcileID":"a931d557-6aa4-44a6-8c0b-4884d155ddc6","job":"test-ns-ptctd/test-trainjob-vkxgp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ptctd"}
{"level":"Level(-2)","ts":"2026-11-02T22:18:06.837101154Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ptctd","name":"test-trainjob-vkxgp","reconcileID":"4409363d-8620-4933-a8a1-4cb20249f989","job":"test-ns-ptctd/test-trainjob-vkxgp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ptctd"}
{"level":"Level(-2)","ts":"2026-11-02T22:18:06.923487909Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ptctd","name":"test-trainjob-vkxgp","reconcileID":"544c273a-bc19-4e6f-9fca-8f2f318b8135","job":"test-ns-ptctd/test-trainjob-vkxgp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ptctd"}
{"level":"Level(-2)","ts":"2026-11-02T22:18:19.637273191Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ptctd","name":"test-trainjob-vkxgp","reconcileID":"6d3cc6df-c20b-4ba7-90f0-8e7729aa876a","job":"test-ns-ptctd/test-trainjob-vkxgp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ptctd"}
{"level":"Level(-2)","ts":"2026-11-02T22:18:19.740090712Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ptctd","name":"test-trainjob-vkxgp","reconcileID":"c6988b55-4dbb-4b4e-ba71-dc295b7417c5","job":"test-ns-ptctd/test-trainjob-vkxgp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ptctd"}
{"level":"Level(-2)","ts":"2026-11-02T22:18:19.74308087Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ptctd","name":"test-trainjob-vkxgp","reconcileID":"a80aac2c-1e3c-4157-b136-0943a654f4cf","job":"test-ns-ptctd/test-trainjob-vkxgp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ptctd"}
{"level":"Level(-2)","ts":"2026-11-02T22:18:19.848741664Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ptctd","name":"test-trainjob-vkxgp","reconcileID":"508d820a-8561-487c-a02b-30c00b6afe9e","job":"test-ns-ptctd/test-trainjob-vkxgp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ptctd"}
{"level":"Level(-2)","ts":"2026-11-02T22:18:25.56541812Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ptctd","name":"test-trainjob-vkxgp","reconcileID":"74375265-ed3e-4118-8438-74db9ddb4234","job":"test-ns-ptctd/test-trainjob-vkxgp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ptctd"}
{"level":"Level(-2)","ts":"2026-11-02T22:18:35.895296617Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-l75px","name":"maaf2d3adaf5","reconcileID":"23628d55-5b6f-405e-a8ae-107816c6d9f1","job":"test-ns-l75px/maaf2d3adaf5","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-l75px"}
{"level":"Level(-2)","ts":"2026-11-02T22:18:36.287596633Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-l75px","name":"maaf2d3adaf5","reconcileID":"6ffd2488-00cd-4cb6-8070-fbe8e1af4558","job":"test-ns-l75px/maaf2d3adaf5","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-l75px"}
{"level":"Level(-2)","ts":"2026-11-02T22:18:36.517666873Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-l75px","name":"maaf2d3adaf5","reconcileID":"af2f6e91-3693-46e8-b695-86be360054f5","job":"test-ns-l75px/maaf2d3adaf5","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-l75px"}
{"level":"Level(-2)","ts":"2026-11-02T22:18:36.621743847Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-l75px","name":"maaf2d3adaf5","reconcileID":"257c3ce9-cd34-4821-87a3-f5da7d008110","job":"test-ns-l75px/maaf2d3adaf5","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-l75px"}
{"level":"Level(-2)","ts":"2026-11-02T22:18:36.728012466Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-l75px","name":"maaf2d3adaf5","reconcileID":"870f15a3-d86b-4033-9329-add17219ed1b","job":"test-ns-l75px/maaf2d3adaf5","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-l75px"}
{"level":"Level(-2)","ts":"2026-11-02T22:18:36.832689156Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-l75px","name":"maaf2d3adaf5","reconcileID":"1cebbf76-c366-4744-8cb0-cb1ac031e376","job":"test-ns-l75px/maaf2d3adaf5","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-l75px"}
{"level":"Level(-2)","ts":"2026-11-02T22:18:39.693514167Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-l75px","name":"maaf2d3adaf5","reconcileID":"209aec04-586b-4f64-b6dd-51d7d0c8e290","job":"test-ns-l75px/maaf2d3adaf5","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-l75px"}
{"level":"Level(-2)","ts":"2026-11-02T22:18:39.798399022Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-l75px","name":"maaf2d3adaf5","reconcileID":"2809ab0b-6bb9-442e-97e3-d814165d5bba","job":"test-ns-l75px/maaf2d3adaf5","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-l75px"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:08.852329236Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-l75px","name":"maaf2d3adaf5","reconcileID":"bc6e9677-87a9-4296-bdb3-1d27befc2986","job":"test-ns-l75px/maaf2d3adaf5","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-l75px"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:08.976350707Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-l75px","name":"maaf2d3adaf5","reconcileID":"9f8e4eb2-0616-4371-aeba-25897759da9a","job":"test-ns-l75px/maaf2d3adaf5","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-l75px"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:09.862984748Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-l75px","name":"maaf2d3adaf5","reconcileID":"19f17f11-2bd4-45c8-bd53-10b9ed0bf087","job":"test-ns-l75px/maaf2d3adaf5","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-l75px"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:09.965393507Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-l75px","name":"maaf2d3adaf5","reconcileID":"5d95f64e-8171-41c3-b485-6f12dd728fe2","job":"test-ns-l75px/maaf2d3adaf5","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-l75px"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:10.023397986Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-l75px","name":"maaf2d3adaf5","reconcileID":"0ba1fde3-4ab9-44a0-bc24-1a0e2dcff2b2","job":"test-ns-l75px/maaf2d3adaf5","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-l75px"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:10.132585729Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-l75px","name":"maaf2d3adaf5","reconcileID":"e77b5ec7-9461-4106-8b80-024051d6870d","job":"test-ns-l75px/maaf2d3adaf5","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-l75px"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:13.556136352Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:141","msg":"ResourceFlavor create event","replica-role":"leader","resourceFlavor":{"name":"rf-zgrws"}}
{"level":"Level(-2)","ts":"2026-11-02T22:19:13.556297523Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-zgrws","reconcileID":"d303f241-7cc7-4e96-8c83-84dcbf484afb"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:13.893571891Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:351","msg":"ClusterQueue create event","replica-role":"leader","clusterQueue":{"name":"cq-chzgj"}}
{"level":"Level(-2)","ts":"2026-11-02T22:19:13.893831118Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-chzgj","reconcileID":"403f41ff-884f-4cb1-8474-d4e20dbb95d2"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:13.911102567Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:240","msg":"LocalQueue create event","replica-role":"leader","localQueue":{"name":"lq-2wmpb","namespace":"test-ns-gwnld"}}
{"level":"Level(-2)","ts":"2026-11-02T22:19:13.911262182Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-gwnld","name":"lq-2wmpb","reconcileID":"a5e0be72-469c-40a4-b0e3-7401b9e4e1e2"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:13.952068553Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-chzgj"}}
{"level":"Level(-2)","ts":"2026-11-02T22:19:13.952232091Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-chzgj","reconcileID":"bf0fcae1-9edb-4044-a4fc-947e6daa1fba"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:13.97013851Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-gwnld","name":"lq-2wmpb","reconcileID":"94da0c49-9d26-43aa-a340-e15756997d2f"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:13.970206745Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-2wmpb","namespace":"test-ns-gwnld"}}
{"level":"Level(-3)","ts":"2026-11-02T22:19:14.024796366Z","logger":"localqueue-reconciler","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-gwnld","name":"lq-2wmpb","reconcileID":"94da0c49-9d26-43aa-a340-e15756997d2f","error":"Operation cannot be fulfilled on localqueues.kueue.x-k8s.io \"lq-2wmpb\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:14.024865739Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-gwnld","name":"lq-2wmpb","reconcileID":"d412035b-5366-4c85-b604-f8fbd275b7c3"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:14.083378599Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-2wmpb","namespace":"test-ns-gwnld"}}
{"level":"Level(-2)","ts":"2026-11-02T22:19:14.08347209Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-gwnld","name":"lq-2wmpb","reconcileID":"4b5f0eab-f7d4-4709-9641-e5634ef94f54"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:14.725369296Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gwnld","name":"dw-deployment-bsjkb-66b85bb665-xpmk8","reconcileID":"61af5cd9-d562-4cea-b53f-ff02aafc7809","job":"test-ns-gwnld/dw-deployment-bsjkb-66b85bb665-xpmk8","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:14.778993672Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1.PriorityClass","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"debug","ts":"2026-11-02T22:19:15.276192737Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"dw-deployment-bsjkb-66b85bb665-xpmk8","namespace":"test-ns-gwnld"},"kind":"Pod","apiVersion":"v1","type":"Normal","reason":"CreatedWorkload","action":"CreatedWorkload","note":"Created Workload: test-ns-gwnld/pod-dw-deployment-bsjkb-66b85bb665-xpmk8-1955b"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:15.276494101Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1276","msg":"Workload create event","replica-role":"leader","workload":{"name":"pod-dw-deployment-bsjkb-66b85bb665-xpmk8-1955b","namespace":"test-ns-gwnld"},"queue":"lq-2wmpb","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:15.276564472Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gwnld","name":"dw-deployment-bsjkb-66b85bb665-xpmk8","reconcileID":"cc9aca62-db30-4b31-a9ab-d862439168d6","job":"test-ns-gwnld/dw-deployment-bsjkb-66b85bb665-xpmk8","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:15.27655086Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-gwnld","name":"pod-dw-deployment-bsjkb-66b85bb665-xpmk8-1955b","reconcileID":"818c9375-991c-4ec8-9266-1199bc266043"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:15.276705149Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-gwnld","name":"pod-dw-deployment-bsjkb-66b85bb665-xpmk8-1955b","reconcileID":"818c9375-991c-4ec8-9266-1199bc266043","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T22:19:15.276758737Z","logger":"scheduler","caller":"scheduler/scheduler.go:308","msg":"Obtained heads","replica-role":"follower","schedulingCycle":1,"headCount":1,"waitDuration":1763.24804928}
{"level":"Level(-2)","ts":"2026-11-02T22:19:15.276803271Z","logger":"scheduler","caller":"scheduler/scheduler.go:323","msg":"Snapshot taken","replica-role":"follower","schedulingCycle":1,"duration":0.000013768}
{"level":"Level(-2)","ts":"2026-11-02T22:19:15.276804582Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-gwnld","name":"pod-dw-deployment-bsjkb-66b85bb665-xpmk8-1955b","reconcileID":"dde5d30d-4d84-43b6-9f09-536243f39718"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:15.279626971Z","logger":"scheduler","caller":"scheduler/scheduler.go:328","msg":"Nomination done","replica-role":"follower","schedulingCycle":1,"entries":1,"inadmissibleEntries":0,"duration":0.002812404}
{"level":"Level(-2)","ts":"2026-11-02T22:19:15.279655852Z","logger":"scheduler","caller":"scheduler/scheduler.go:406","msg":"Attempting to schedule workload","replica-role":"follower","schedulingCycle":1,"workload":{"name":"pod-dw-deployment-bsjkb-66b85bb665-xpmk8-1955b","namespace":"test-ns-gwnld"},"clusterQueue":{"name":"cq-chzgj"}}
{"level":"Level(-2)","ts":"2026-11-02T22:19:15.279782914Z","logger":"scheduler","caller":"scheduler/scheduler.go:1041","msg":"Workload assumed in the cache","replica-role":"follower","schedulingCycle":1,"workload":{"name":"pod-dw-deployment-bsjkb-66b85bb665-xpmk8-1955b","namespace":"test-ns-gwnld"},"clusterQueue":{"name":"cq-chzgj"}}
{"level":"Level(-2)","ts":"2026-11-02T22:19:15.282218075Z","logger":"scheduler","caller":"scheduler/scheduler.go:362","msg":"Workload processing done","replica-role":"follower","schedulingCycle":1,"duration":0.002565938}
{"level":"Level(-2)","ts":"2026-11-02T22:19:15.282259573Z","logger":"scheduler","caller":"scheduler/scheduler.go:297","msg":"Scheduling cycle complete","replica-role":"follower","schedulingCycle":1,"duration":1763.253551229}
{"level":"Level(-2)","ts":"2026-11-02T22:19:15.282276675Z","logger":"scheduler","caller":"scheduler/scheduler.go:295","msg":"Scheduling cycle starts","replica-role":"leader","schedulingCycle":2}
{"level":"Level(-2)","ts":"2026-11-02T22:19:15.397273229Z","logger":"scheduler","caller":"scheduler/scheduler.go:1000","msg":"Workload successfully admitted and assigned flavors","replica-role":"follower","schedulingCycle":1,"workload":{"name":"pod-dw-deployment-bsjkb-66b85bb665-xpmk8-1955b","namespace":"test-ns-gwnld"},"clusterQueue":{"name":"cq-chzgj"},"assignments":[{"name":"main","flavors":{"cpu":"rf-zgrws","memory":"rf-zgrws"},"resourceUsage":{"cpu":"1","memory":"3Gi"},"count":1}]}
{"level":"debug","ts":"2026-11-02T22:19:15.397328898Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"pod-dw-deployment-bsjkb-66b85bb665-xpmk8-1955b","namespace":"test-ns-gwnld"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"QuotaReserved","action":"QuotaReserved","note":"Quota reserved in ClusterQueue cq-chzgj, wait time since queued was 0s"}
{"level":"debug","ts":"2026-11-02T22:19:15.397363518Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"pod-dw-deployment-bsjkb-66b85bb665-xpmk8-1955b","namespace":"test-ns-gwnld"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"Admitted","action":"Admitted","note":"Admitted by ClusterQueue cq-chzgj, wait time since reservation was 0s"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:15.397727614Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"pod-dw-deployment-bsjkb-66b85bb665-xpmk8-1955b","namespace":"test-ns-gwnld"},"queue":"lq-2wmpb","status":"admitted","prevStatus":"pending","clusterQueue":"cq-chzgj"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:15.397763239Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-gwnld","name":"pod-dw-deployment-bsjkb-66b85bb665-xpmk8-1955b","reconcileID":"51ecc5f5-0e2c-433a-b22c-f2204494e080"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:15.397816227Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gwnld","name":"dw-deployment-bsjkb-66b85bb665-xpmk8","reconcileID":"b60dd457-5d92-41b8-99b4-72a56f1448fd","job":"test-ns-gwnld/dw-deployment-bsjkb-66b85bb665-xpmk8","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:15.397829945Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-gwnld","name":"pod-dw-deployment-bsjkb-66b85bb665-xpmk8-1955b","reconcileID":"51ecc5f5-0e2c-433a-b22c-f2204494e080","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T22:19:15.397974193Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-gwnld","name":"pod-dw-deployment-bsjkb-66b85bb665-xpmk8-1955b","reconcileID":"eb82628f-0422-4f0d-8798-293e395778f3"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:15.398033755Z","logger":"v1_pod","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-gwnld","name":"dw-deployment-bsjkb-66b85bb665-xpmk8","reconcileID":"b60dd457-5d92-41b8-99b4-72a56f1448fd","job":"test-ns-gwnld/dw-deployment-bsjkb-66b85bb665-xpmk8","gvk":"/v1, Kind=Pod"}
{"level":"debug","ts":"2026-11-02T22:19:15.514515076Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"dw-deployment-bsjkb-66b85bb665-xpmk8","namespace":"test-ns-gwnld"},"kind":"Pod","apiVersion":"v1","type":"Normal","reason":"Started","action":"Started","note":"Admitted by clusterQueue cq-chzgj"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:15.514608807Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gwnld","name":"dw-deployment-bsjkb-66b85bb665-xpmk8","reconcileID":"9b13d087-71c7-425b-a117-f9486117d441","job":"test-ns-gwnld/dw-deployment-bsjkb-66b85bb665-xpmk8","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:15.521676465Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gwnld","name":"dw-deployment-bsjkb-66b85bb665-xpmk8","reconcileID":"412bc74b-abad-468d-80ac-8d7c15c502bd","job":"test-ns-gwnld/dw-deployment-bsjkb-66b85bb665-xpmk8","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:15.580283235Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gwnld","name":"dw-deployment-bsjkb-66b85bb665-xpmk8","reconcileID":"ca1ea880-2640-4bf0-95d8-6bc0b88c767a","job":"test-ns-gwnld/dw-deployment-bsjkb-66b85bb665-xpmk8","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:15.733372074Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gwnld","name":"dw-deployment-bsjkb-66b85bb665-xpmk8","reconcileID":"c9443600-b324-4eda-afd3-383fa718e15a","job":"test-ns-gwnld/dw-deployment-bsjkb-66b85bb665-xpmk8","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:16.277532082Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-gwnld","name":"lq-2wmpb","reconcileID":"3924196b-8ec4-466f-b33d-3957ea42a506"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:16.277535131Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-chzgj","reconcileID":"3d2fe73d-83a4-4de8-8e8e-1c4f666e8717"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:16.337087917Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-2wmpb","namespace":"test-ns-gwnld"}}
{"level":"Level(-2)","ts":"2026-11-02T22:19:16.337188649Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-gwnld","name":"lq-2wmpb","reconcileID":"ddbac01b-bd47-4cf1-b4fa-b4a28e7128e5"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:16.338045983Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-chzgj"}}
{"level":"Level(-2)","ts":"2026-11-02T22:19:16.338169056Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-chzgj","reconcileID":"1937a31b-cec8-4d60-bdda-62872418fc8b"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:16.513781063Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gwnld","name":"dw-deployment-bsjkb-66b85bb665-xpmk8","reconcileID":"9fe96728-9478-4853-a0ea-1f625a72fd10","job":"test-ns-gwnld/dw-deployment-bsjkb-66b85bb665-xpmk8","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:16.515674134Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-gwnld","name":"pod-dw-deployment-bsjkb-66b85bb665-xpmk8-1955b","reconcileID":"8e11c426-fa10-4925-9156-27c7c274125b"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:17.786756289Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gwnld","name":"dw-deployment-bsjkb-66b85bb665-xpmk8","reconcileID":"7fcbd3c5-0bb2-4caa-bbf9-e2d5f05efb08","job":"test-ns-gwnld/dw-deployment-bsjkb-66b85bb665-xpmk8","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:18.787455426Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-gwnld","name":"pod-dw-deployment-bsjkb-66b85bb665-xpmk8-1955b","reconcileID":"5a6b0979-8c23-46f8-871b-e8328c711d87"}
{"level":"info","ts":"2026-11-02T22:19:30.161780997Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:117","msg":"Validating create","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"x246af69c099","namespace":"test-ns-gwnld"},"namespace":"test-ns-gwnld","name":"x246af69c099","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:redhat-ods-applications:trainer-e2e","requestID":"b3ddfa27-529f-4cfc-a2d7-8c9be0673e01","replica-role":"leader"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:30.223229927Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gwnld","name":"x246af69c099","reconcileID":"f427bf67-d4f4-4216-9335-534c0826cda0","job":"test-ns-gwnld/x246af69c099","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"debug","ts":"2026-11-02T22:19:30.547023568Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"x246af69c099","namespace":"test-ns-gwnld"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"CreatedWorkload","action":"CreatedWorkload","note":"Created Workload: test-ns-gwnld/trainjob-x246af69c099-74af4"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:30.547055272Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gwnld","name":"x246af69c099","reconcileID":"1a5af4f8-8d36-4fd0-a712-81a7d0172730","job":"test-ns-gwnld/x246af69c099","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:30.547154274Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1276","msg":"Workload create event","replica-role":"leader","workload":{"name":"trainjob-x246af69c099-74af4","namespace":"test-ns-gwnld"},"queue":"lq-2wmpb","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:30.547252076Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-gwnld","name":"trainjob-x246af69c099-74af4","reconcileID":"47cb6696-cb59-40c1-b241-f4954dea3e00"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:30.547300767Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-gwnld","name":"trainjob-x246af69c099-74af4","reconcileID":"47cb6696-cb59-40c1-b241-f4954dea3e00","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T22:19:30.547345211Z","logger":"scheduler","caller":"scheduler/scheduler.go:308","msg":"Obtained heads","replica-role":"leader","schedulingCycle":2,"headCount":1,"waitDuration":15.26506792}
{"level":"Level(-2)","ts":"2026-11-02T22:19:30.547381198Z","logger":"scheduler","caller":"scheduler/scheduler.go:323","msg":"Snapshot taken","replica-role":"leader","schedulingCycle":2,"duration":0.00001427}
{"level":"Level(-2)","ts":"2026-11-02T22:19:30.547407385Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-gwnld","name":"trainjob-x246af69c099-74af4","reconcileID":"fc697843-8288-427b-a3fa-aaa2bf0d98fc"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:30.547462338Z","logger":"scheduler","caller":"scheduler/scheduler.go:328","msg":"Nomination done","replica-role":"leader","schedulingCycle":2,"entries":1,"inadmissibleEntries":0,"duration":0.000071576}
{"level":"Level(-2)","ts":"2026-11-02T22:19:30.547486061Z","logger":"scheduler","caller":"scheduler/scheduler.go:406","msg":"Attempting to schedule workload","replica-role":"leader","schedulingCycle":2,"workload":{"name":"trainjob-x246af69c099-74af4","namespace":"test-ns-gwnld"},"clusterQueue":{"name":"cq-chzgj"}}
{"level":"Level(-2)","ts":"2026-11-02T22:19:30.547577634Z","logger":"scheduler","caller":"scheduler/scheduler.go:1041","msg":"Workload assumed in the cache","replica-role":"leader","schedulingCycle":2,"workload":{"name":"trainjob-x246af69c099-74af4","namespace":"test-ns-gwnld"},"clusterQueue":{"name":"cq-chzgj"}}
{"level":"Level(-2)","ts":"2026-11-02T22:19:30.547606224Z","logger":"scheduler","caller":"scheduler/scheduler.go:362","msg":"Workload processing done","replica-role":"leader","schedulingCycle":2,"duration":0.000125681}
{"level":"Level(-2)","ts":"2026-11-02T22:19:30.547624576Z","logger":"scheduler","caller":"scheduler/scheduler.go:297","msg":"Scheduling cycle complete","replica-role":"leader","schedulingCycle":2,"duration":15.265347451}
{"level":"Level(-2)","ts":"2026-11-02T22:19:30.54764372Z","logger":"scheduler","caller":"scheduler/scheduler.go:295","msg":"Scheduling cycle starts","replica-role":"leader","schedulingCycle":3}
{"level":"Level(-2)","ts":"2026-11-02T22:19:30.658667613Z","logger":"scheduler","caller":"scheduler/scheduler.go:1000","msg":"Workload successfully admitted and assigned flavors","replica-role":"leader","schedulingCycle":2,"workload":{"name":"trainjob-x246af69c099-74af4","namespace":"test-ns-gwnld"},"clusterQueue":{"name":"cq-chzgj"},"assignments":[{"name":"node","flavors":{"cpu":"rf-zgrws","memory":"rf-zgrws"},"resourceUsage":{"cpu":"4","memory":"16Gi"},"count":2}]}
{"level":"debug","ts":"2026-11-02T22:19:30.658714883Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-x246af69c099-74af4","namespace":"test-ns-gwnld"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"Admitted","action":"Admitted","note":"Admitted by ClusterQueue cq-chzgj, wait time since reservation was 1s"}
{"level":"debug","ts":"2026-11-02T22:19:30.658737543Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-x246af69c099-74af4","namespace":"test-ns-gwnld"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"QuotaReserved","action":"QuotaReserved","note":"Quota reserved in ClusterQueue cq-chzgj, wait time since queued was 1s"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:30.658870475Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-x246af69c099-74af4","namespace":"test-ns-gwnld"},"queue":"lq-2wmpb","status":"admitted","prevStatus":"pending","clusterQueue":"cq-chzgj"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:30.658872335Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-gwnld","name":"trainjob-x246af69c099-74af4","reconcileID":"b980794c-6bf2-4e42-87ed-2b4e49a188a6"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:30.658927989Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-gwnld","name":"trainjob-x246af69c099-74af4","reconcileID":"b980794c-6bf2-4e42-87ed-2b4e49a188a6","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T22:19:30.659118526Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-gwnld","name":"trainjob-x246af69c099-74af4","reconcileID":"d7a3101b-6622-474e-a5c6-a6f0487798d9"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:30.705585601Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gwnld","name":"x246af69c099","reconcileID":"67937b02-d008-4c90-a487-7e8ba28325fc","job":"test-ns-gwnld/x246af69c099","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:30.864205588Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-gwnld","name":"x246af69c099","reconcileID":"67937b02-d008-4c90-a487-7e8ba28325fc","job":"test-ns-gwnld/x246af69c099","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"info","ts":"2026-11-02T22:19:30.924114035Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"x246af69c099","namespace":"test-ns-gwnld"},"namespace":"test-ns-gwnld","name":"x246af69c099","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"194a084f-fe5c-4547-9fb0-cd13fcda2412","replica-role":"leader"}
{"level":"info","ts":"2026-11-02T22:19:31.042919036Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"x246af69c099","namespace":"test-ns-gwnld"},"namespace":"test-ns-gwnld","name":"x246af69c099","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"30199f44-5ab9-4bc5-a054-76e82741c5ff","replica-role":"leader"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:31.101129782Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gwnld","name":"x246af69c099","reconcileID":"5a338e06-0f70-4d9b-a571-eca09429b6aa","job":"test-ns-gwnld/x246af69c099","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"debug","ts":"2026-11-02T22:19:31.101136152Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"x246af69c099","namespace":"test-ns-gwnld"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"Started","action":"Started","note":"Admitted by clusterQueue cq-chzgj"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:31.260511992Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-gwnld","name":"x246af69c099","reconcileID":"5a338e06-0f70-4d9b-a571-eca09429b6aa","job":"test-ns-gwnld/x246af69c099","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-3)","ts":"2026-11-02T22:19:31.31806321Z","logger":"trainjob","caller":"jobframework/reconciler.go:594","msg":"Unsuspending job","replica-role":"leader","namespace":"test-ns-gwnld","name":"x246af69c099","reconcileID":"5a338e06-0f70-4d9b-a571-eca09429b6aa","job":"test-ns-gwnld/x246af69c099","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"x246af69c099\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-3)","ts":"2026-11-02T22:19:31.318143963Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-gwnld","name":"x246af69c099","reconcileID":"5a338e06-0f70-4d9b-a571-eca09429b6aa","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"x246af69c099\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:31.318237049Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gwnld","name":"x246af69c099","reconcileID":"49bd4165-b013-44f3-bb65-3ed38f41f899","job":"test-ns-gwnld/x246af69c099","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:31.477285823Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gwnld","name":"x246af69c099","reconcileID":"9280f8d1-5998-4317-8812-9d651388b7ea","job":"test-ns-gwnld/x246af69c099","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:31.548317288Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-chzgj","reconcileID":"8e4dca22-e1c2-4287-b151-79b61a052fef"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:31.548328787Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-gwnld","name":"lq-2wmpb","reconcileID":"5d4dfe87-1742-4d3f-9f04-f0bb3397f6ca"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:31.605645875Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-2wmpb","namespace":"test-ns-gwnld"}}
{"level":"Level(-2)","ts":"2026-11-02T22:19:31.605738093Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-gwnld","name":"lq-2wmpb","reconcileID":"bc98ce9a-3544-4975-9ddc-7a842afa048b"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:31.60866212Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-chzgj"}}
{"level":"Level(-2)","ts":"2026-11-02T22:19:31.608811334Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-chzgj","reconcileID":"4d9ec3ac-9d6b-4644-b815-01c5c9e34db8"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:31.63673479Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gwnld","name":"x246af69c099","reconcileID":"d8986ecc-313c-4dd6-83b6-55ab88ca3f64","job":"test-ns-gwnld/x246af69c099","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:31.796141639Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gwnld","name":"x246af69c099","reconcileID":"6932070a-1276-4362-bc6d-03190f05a42f","job":"test-ns-gwnld/x246af69c099","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:32.126378236Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gwnld","name":"x246af69c099","reconcileID":"ff178bec-2c32-41b2-9a9a-991cf16bbf92","job":"test-ns-gwnld/x246af69c099","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:32.285941345Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gwnld","name":"x246af69c099","reconcileID":"9693ac3a-0a86-42ef-94c9-f836eda9a77e","job":"test-ns-gwnld/x246af69c099","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:32.878320977Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-gwnld","name":"trainjob-x246af69c099-74af4","reconcileID":"49f58fc1-0256-499b-95b7-767770aa0ed9"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:33.963466498Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-gwnld","name":"trainjob-x246af69c099-74af4","reconcileID":"567b1c3a-2272-4676-9e6d-25216ee9800d"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:34.984008993Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-gwnld","name":"trainjob-x246af69c099-74af4","reconcileID":"9776c803-b282-4193-b119-27220dff5942"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:35.048558599Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gwnld","name":"x246af69c099","reconcileID":"c6fe3853-67c8-49cc-a325-156cc80c0afe","job":"test-ns-gwnld/x246af69c099","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:19:35.252723699Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gwnld","name":"x246af69c099","reconcileID":"b31ac5ba-0510-491b-bc28-3d73de292805","job":"test-ns-gwnld/x246af69c099","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:05.180938119Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-gwnld","name":"trainjob-x246af69c099-74af4","reconcileID":"33e6be18-7049-4e3d-a0fc-4f7cb05d8c80"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:05.203516941Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gwnld","name":"x246af69c099","reconcileID":"61904c53-b4c5-4208-8d15-744fc6529f84","job":"test-ns-gwnld/x246af69c099","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:05.36406281Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gwnld","name":"x246af69c099","reconcileID":"044c9b66-7360-4f34-b618-44cfb912fc3c","job":"test-ns-gwnld/x246af69c099","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:06.213566122Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gwnld","name":"x246af69c099","reconcileID":"97f72dbd-7bd7-4baa-958d-f5a292d92517","job":"test-ns-gwnld/x246af69c099","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:06.254835632Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-gwnld","name":"trainjob-x246af69c099-74af4","reconcileID":"faba2553-7f63-4ed3-bda8-973eacae3917"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:06.373116327Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gwnld","name":"x246af69c099","reconcileID":"1f595c78-3b8f-448d-82fc-fff28a4593a2","job":"test-ns-gwnld/x246af69c099","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:06.532787761Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gwnld","name":"x246af69c099","reconcileID":"267c81ed-9f0c-420b-b1bb-f5856bbf21e9","job":"test-ns-gwnld/x246af69c099","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"debug","ts":"2026-11-02T22:20:06.804315404Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"x246af69c099","namespace":"test-ns-gwnld"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"FinishedWorkload","action":"FinishedWorkload","note":"Workload 'test-ns-gwnld/trainjob-x246af69c099-74af4' is declared finished"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:06.804771585Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-x246af69c099-74af4","namespace":"test-ns-gwnld"},"queue":"lq-2wmpb","status":"finished","prevStatus":"admitted","clusterQueue":"cq-chzgj"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:06.804775396Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-gwnld","name":"trainjob-x246af69c099-74af4","reconcileID":"98eb126f-ecc3-4b20-ae6a-1886746357aa"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:06.804833032Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-gwnld","name":"trainjob-x246af69c099-74af4","reconcileID":"98eb126f-ecc3-4b20-ae6a-1886746357aa","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T22:20:06.804969321Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gwnld","name":"x246af69c099","reconcileID":"f07f2178-c4cf-4aa1-a8d3-73e704fe8285","job":"test-ns-gwnld/x246af69c099","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:06.805009282Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-gwnld","name":"trainjob-x246af69c099-74af4","reconcileID":"7f4f0dae-afd9-44e1-96d4-6cbb72bc50d6"}
{"level":"debug","ts":"2026-11-02T22:20:06.96453243Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"x246af69c099","namespace":"test-ns-gwnld"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"FinishedWorkload","action":"FinishedWorkload","note":"Workload 'test-ns-gwnld/trainjob-x246af69c099-74af4' is declared finished"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:07.076564474Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-x246af69c099-74af4","namespace":"test-ns-gwnld"},"queue":"lq-2wmpb","status":"finished","clusterQueue":"cq-chzgj"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:07.0765922Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-gwnld","name":"trainjob-x246af69c099-74af4","reconcileID":"df8b892f-4f86-4d63-ac1e-f24b9ceffa69"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:07.076656978Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-gwnld","name":"trainjob-x246af69c099-74af4","reconcileID":"df8b892f-4f86-4d63-ac1e-f24b9ceffa69","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T22:20:07.076714406Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gwnld","name":"x246af69c099","reconcileID":"f435d8d9-3fe4-4b43-8101-a9591c3b4da4","job":"test-ns-gwnld/x246af69c099","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:07.076725127Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-gwnld","name":"trainjob-x246af69c099-74af4","reconcileID":"b64ef50c-0c75-4f9a-9b7d-d863d3b0a1e6"}
{"level":"debug","ts":"2026-11-02T22:20:07.236320991Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"x246af69c099","namespace":"test-ns-gwnld"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"FinishedWorkload","action":"FinishedWorkload","note":"Workload 'test-ns-gwnld/trainjob-x246af69c099-74af4' is declared finished"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:07.265275039Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-gwnld","name":"trainjob-x246af69c099-74af4","reconcileID":"70cd8566-1670-4dec-bb2b-ae25fc5e46a1"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:07.805066549Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-gwnld","name":"lq-2wmpb","reconcileID":"95f343e0-222e-4d69-b611-78a1ec58de40"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:07.805066635Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-chzgj","reconcileID":"626c181c-d6ad-4907-8235-fe2d358634cc"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:07.862539273Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-2wmpb","namespace":"test-ns-gwnld"}}
{"level":"Level(-2)","ts":"2026-11-02T22:20:07.86266592Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-gwnld","name":"lq-2wmpb","reconcileID":"907f0bc4-9e93-4c06-9a24-474314f045bc"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:07.865670883Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-chzgj"}}
{"level":"Level(-2)","ts":"2026-11-02T22:20:07.865820565Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-chzgj","reconcileID":"8d1e23ec-2449-455e-8096-3422398b8219"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:08.826399513Z","logger":"trainjob","caller":"workload/workload.go:1810","msg":"Workload is orphaned; finishing to release quota","replica-role":"leader","namespace":"test-ns-gwnld","name":"x246af69c099","reconcileID":"ae9d513e-e3eb-4c3f-9cec-b8bb8f5675c9","job":"test-ns-gwnld/x246af69c099","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:08.878091166Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1312","msg":"Workload delete event","replica-role":"leader","workload":{"name":"trainjob-x246af69c099-74af4","namespace":"test-ns-gwnld"},"queue":"lq-2wmpb","status":"finished"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:08.878095728Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-gwnld","name":"trainjob-x246af69c099-74af4","reconcileID":"1b894023-2e2c-47aa-952a-64923dee8a6d"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:08.878151516Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-gwnld","name":"trainjob-x246af69c099-74af4","reconcileID":"1b894023-2e2c-47aa-952a-64923dee8a6d","isDeleted":true}
{"level":"Level(-2)","ts":"2026-11-02T22:20:09.878282769Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-chzgj","reconcileID":"3f741c2e-e550-42b5-9b8b-6db83026e6b9"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:09.878279506Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-gwnld","name":"lq-2wmpb","reconcileID":"c8f8d5e2-3db2-463d-9311-5b5d517a333e"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:09.952691643Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-chzgj"}}
{"level":"Level(-2)","ts":"2026-11-02T22:20:09.952797812Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-chzgj","reconcileID":"9c1f3f26-1021-4ed0-873f-0869f54eec23"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:09.952844725Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-gwnld","name":"pod-dw-deployment-bsjkb-66b85bb665-xpmk8-1955b","reconcileID":"ad2d275e-eee6-48b3-983d-cf2cb27d79c4"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:09.969266298Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:171","msg":"ResourceFlavor update event","replica-role":"leader","resourceFlavor":{"name":"rf-zgrws"}}
{"level":"Level(-2)","ts":"2026-11-02T22:20:09.969376557Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-zgrws","reconcileID":"c7b3baaf-e69c-4e32-a3f0-2eeef1c434ec"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:10.011103758Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-chzgj"}}
{"level":"Level(-2)","ts":"2026-11-02T22:20:10.011161092Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-chzgj","reconcileID":"d8edb1ff-f66a-4af3-9032-fd1c2401e2d3"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:10.011182305Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-gwnld","name":"lq-2wmpb","reconcileID":"abc6b1bc-69fd-431a-b6aa-e0ecad3fcdf2"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:10.011217262Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-gwnld","name":"pod-dw-deployment-bsjkb-66b85bb665-xpmk8-1955b","reconcileID":"3b29291e-6599-4bd1-b31b-81f359f8de5e"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:10.068989332Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-2wmpb","namespace":"test-ns-gwnld"}}
{"level":"Level(-2)","ts":"2026-11-02T22:20:10.069110043Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-gwnld","name":"lq-2wmpb","reconcileID":"7e3a05f1-1baa-4c93-8d77-9c3e3a71b9ef"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:10.344706439Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:141","msg":"ResourceFlavor create event","replica-role":"leader","resourceFlavor":{"name":"rf-qrb5g"}}
{"level":"Level(-2)","ts":"2026-11-02T22:20:10.344813177Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-qrb5g","reconcileID":"e715669a-781f-478f-be2d-ae469b00dec5"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:10.461394452Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:351","msg":"ClusterQueue create event","replica-role":"leader","clusterQueue":{"name":"cq-lqbf7"}}
{"level":"Level(-2)","ts":"2026-11-02T22:20:10.461624221Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-lqbf7","reconcileID":"78e4262e-f8e6-4b11-9284-0f651c116eb2"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:10.478489061Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:240","msg":"LocalQueue create event","replica-role":"leader","localQueue":{"name":"lq-vj289","namespace":"test-ns-7bgbq"}}
{"level":"Level(-2)","ts":"2026-11-02T22:20:10.478645748Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-7bgbq","name":"lq-vj289","reconcileID":"67ff4b89-c1f0-4cb7-b351-0addff628607"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:10.521443094Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-lqbf7"}}
{"level":"Level(-2)","ts":"2026-11-02T22:20:10.521610198Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-lqbf7","reconcileID":"cf1d7e5b-4f5b-4707-8462-7f4db2fc8d29"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:10.538010209Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-7bgbq","name":"lq-vj289","reconcileID":"6718d9f2-7d3d-4dd0-8f1c-714c3a835880"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:10.538039879Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-vj289","namespace":"test-ns-7bgbq"}}
{"level":"info","ts":"2026-11-02T22:20:10.543059911Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:117","msg":"Validating create","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-preemption-trainjob-xmrql","namespace":"test-ns-7bgbq"},"namespace":"test-ns-7bgbq","name":"test-preemption-trainjob-xmrql","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:admin","requestID":"3f3dd4ca-5d92-4a49-a85f-d33e0c8a771c","replica-role":"leader"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:10.597493595Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-7bgbq","name":"lq-vj289","reconcileID":"491f66a6-7738-4d2b-9ea3-22fbee779f93"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:10.597521704Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-vj289","namespace":"test-ns-7bgbq"}}
{"level":"Level(-2)","ts":"2026-11-02T22:20:10.597605914Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-7bgbq","name":"lq-vj289","reconcileID":"f9c6b67e-37df-4169-961e-2dc05ef18d35"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:10.599754884Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-7bgbq","name":"test-preemption-trainjob-xmrql","reconcileID":"8cd296d6-ca16-4490-9f3e-bf73c71bd101","job":"test-ns-7bgbq/test-preemption-trainjob-xmrql","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:10.914681964Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1276","msg":"Workload create event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-xmrql-e533e","namespace":"test-ns-7bgbq"},"queue":"lq-vj289","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:10.914757405Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-7bgbq","name":"trainjob-test-preemption-trainjob-xmrql-e533e","reconcileID":"f5027da9-021f-4a07-aab2-4bcfac0cffb7"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:10.914813734Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-7bgbq","name":"trainjob-test-preemption-trainjob-xmrql-e533e","reconcileID":"f5027da9-021f-4a07-aab2-4bcfac0cffb7","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T22:20:10.914812662Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-7bgbq","name":"test-preemption-trainjob-xmrql","reconcileID":"b2848828-8de4-4a86-b977-5b789de619ed","job":"test-ns-7bgbq/test-preemption-trainjob-xmrql","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"debug","ts":"2026-11-02T22:20:10.914820696Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"test-preemption-trainjob-xmrql","namespace":"test-ns-7bgbq"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"CreatedWorkload","action":"CreatedWorkload","note":"Created Workload: test-ns-7bgbq/trainjob-test-preemption-trainjob-xmrql-e533e"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:10.91486093Z","logger":"scheduler","caller":"scheduler/scheduler.go:308","msg":"Obtained heads","replica-role":"leader","schedulingCycle":3,"headCount":1,"waitDuration":40.36721639}
{"level":"Level(-2)","ts":"2026-11-02T22:20:10.914891065Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-7bgbq","name":"trainjob-test-preemption-trainjob-xmrql-e533e","reconcileID":"a3480f77-f658-4b1f-9105-bee64967e1cf"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:10.91489822Z","logger":"scheduler","caller":"scheduler/scheduler.go:323","msg":"Snapshot taken","replica-role":"leader","schedulingCycle":3,"duration":0.00001301}
{"level":"Level(-2)","ts":"2026-11-02T22:20:10.915006017Z","logger":"scheduler","caller":"scheduler/scheduler.go:328","msg":"Nomination done","replica-role":"leader","schedulingCycle":3,"entries":1,"inadmissibleEntries":0,"duration":0.000096052}
{"level":"Level(-2)","ts":"2026-11-02T22:20:10.91503371Z","logger":"scheduler","caller":"scheduler/scheduler.go:406","msg":"Attempting to schedule workload","replica-role":"leader","schedulingCycle":3,"workload":{"name":"trainjob-test-preemption-trainjob-xmrql-e533e","namespace":"test-ns-7bgbq"},"clusterQueue":{"name":"cq-lqbf7"}}
{"level":"Level(-2)","ts":"2026-11-02T22:20:10.915111695Z","logger":"scheduler","caller":"scheduler/scheduler.go:1041","msg":"Workload assumed in the cache","replica-role":"leader","schedulingCycle":3,"workload":{"name":"trainjob-test-preemption-trainjob-xmrql-e533e","namespace":"test-ns-7bgbq"},"clusterQueue":{"name":"cq-lqbf7"}}
{"level":"Level(-2)","ts":"2026-11-02T22:20:10.915145502Z","logger":"scheduler","caller":"scheduler/scheduler.go:362","msg":"Workload processing done","replica-role":"leader","schedulingCycle":3,"duration":0.000117157}
{"level":"Level(-2)","ts":"2026-11-02T22:20:10.915162954Z","logger":"scheduler","caller":"scheduler/scheduler.go:297","msg":"Scheduling cycle complete","replica-role":"leader","schedulingCycle":3,"duration":40.367518877}
{"level":"Level(-2)","ts":"2026-11-02T22:20:10.915180069Z","logger":"scheduler","caller":"scheduler/scheduler.go:295","msg":"Scheduling cycle starts","replica-role":"leader","schedulingCycle":4}
{"level":"Level(-2)","ts":"2026-11-02T22:20:10.933695966Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-gwnld","name":"pod-dw-deployment-bsjkb-66b85bb665-xpmk8-1955b","reconcileID":"88205ec8-1e7c-4220-a606-27772b7cebfa"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:11.023840187Z","logger":"scheduler","caller":"scheduler/scheduler.go:1000","msg":"Workload successfully admitted and assigned flavors","replica-role":"leader","schedulingCycle":3,"workload":{"name":"trainjob-test-preemption-trainjob-xmrql-e533e","namespace":"test-ns-7bgbq"},"clusterQueue":{"name":"cq-lqbf7"},"assignments":[{"name":"node","count":1}]}
{"level":"Level(-2)","ts":"2026-11-02T22:20:11.023863868Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-xmrql-e533e","namespace":"test-ns-7bgbq"},"queue":"lq-vj289","status":"admitted","prevStatus":"pending","clusterQueue":"cq-lqbf7"}
{"level":"debug","ts":"2026-11-02T22:20:11.023846578Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-test-preemption-trainjob-xmrql-e533e","namespace":"test-ns-7bgbq"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"QuotaReserved","action":"QuotaReserved","note":"Quota reserved in ClusterQueue cq-lqbf7, wait time since queued was 1s"}
{"level":"debug","ts":"2026-11-02T22:20:11.023899531Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-test-preemption-trainjob-xmrql-e533e","namespace":"test-ns-7bgbq"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"Admitted","action":"Admitted","note":"Admitted by ClusterQueue cq-lqbf7, wait time since reservation was 1s"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:11.023933466Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-7bgbq","name":"trainjob-test-preemption-trainjob-xmrql-e533e","reconcileID":"a34f0a9f-c723-422d-b11e-0af32e18c830"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:11.024034319Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-7bgbq","name":"trainjob-test-preemption-trainjob-xmrql-e533e","reconcileID":"a34f0a9f-c723-422d-b11e-0af32e18c830","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T22:20:11.024071762Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-7bgbq","name":"trainjob-test-preemption-trainjob-xmrql-e533e","reconcileID":"d5567d3e-5fd6-409c-8ad4-f3110da6cfd5"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:11.072250649Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-7bgbq","name":"test-preemption-trainjob-xmrql","reconcileID":"3a6653d1-3104-4054-9c76-1e409d46a822","job":"test-ns-7bgbq/test-preemption-trainjob-xmrql","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:11.229143024Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-7bgbq","name":"test-preemption-trainjob-xmrql","reconcileID":"3a6653d1-3104-4054-9c76-1e409d46a822","job":"test-ns-7bgbq/test-preemption-trainjob-xmrql","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-3)","ts":"2026-11-02T22:20:11.284875225Z","logger":"trainjob","caller":"jobframework/reconciler.go:594","msg":"Unsuspending job","replica-role":"leader","namespace":"test-ns-7bgbq","name":"test-preemption-trainjob-xmrql","reconcileID":"3a6653d1-3104-4054-9c76-1e409d46a822","job":"test-ns-7bgbq/test-preemption-trainjob-xmrql","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-preemption-trainjob-xmrql\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-3)","ts":"2026-11-02T22:20:11.284941461Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-7bgbq","name":"test-preemption-trainjob-xmrql","reconcileID":"3a6653d1-3104-4054-9c76-1e409d46a822","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-preemption-trainjob-xmrql\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:11.285044727Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-7bgbq","name":"test-preemption-trainjob-xmrql","reconcileID":"24f0b1b4-3d97-4c5c-accc-5340084b332b","job":"test-ns-7bgbq/test-preemption-trainjob-xmrql","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:11.442217294Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-7bgbq","name":"test-preemption-trainjob-xmrql","reconcileID":"24f0b1b4-3d97-4c5c-accc-5340084b332b","job":"test-ns-7bgbq/test-preemption-trainjob-xmrql","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-3)","ts":"2026-11-02T22:20:11.498156617Z","logger":"trainjob","caller":"jobframework/reconciler.go:594","msg":"Unsuspending job","replica-role":"leader","namespace":"test-ns-7bgbq","name":"test-preemption-trainjob-xmrql","reconcileID":"24f0b1b4-3d97-4c5c-accc-5340084b332b","job":"test-ns-7bgbq/test-preemption-trainjob-xmrql","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-preemption-trainjob-xmrql\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-3)","ts":"2026-11-02T22:20:11.498230518Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-7bgbq","name":"test-preemption-trainjob-xmrql","reconcileID":"24f0b1b4-3d97-4c5c-accc-5340084b332b","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-preemption-trainjob-xmrql\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:11.49833142Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-7bgbq","name":"test-preemption-trainjob-xmrql","reconcileID":"090feb72-9327-46e8-81a4-46fd46c74da2","job":"test-ns-7bgbq/test-preemption-trainjob-xmrql","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:11.655387939Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-7bgbq","name":"test-preemption-trainjob-xmrql","reconcileID":"090feb72-9327-46e8-81a4-46fd46c74da2","job":"test-ns-7bgbq/test-preemption-trainjob-xmrql","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"info","ts":"2026-11-02T22:20:11.712839198Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-preemption-trainjob-xmrql","namespace":"test-ns-7bgbq"},"namespace":"test-ns-7bgbq","name":"test-preemption-trainjob-xmrql","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"71817745-6666-4720-b91b-b4ec7255e8be","replica-role":"leader"}
{"level":"info","ts":"2026-11-02T22:20:11.828116821Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-preemption-trainjob-xmrql","namespace":"test-ns-7bgbq"},"namespace":"test-ns-7bgbq","name":"test-preemption-trainjob-xmrql","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"00cafb48-ae05-4236-9a28-5e4889868462","replica-role":"leader"}
{"level":"debug","ts":"2026-11-02T22:20:11.885549635Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"test-preemption-trainjob-xmrql","namespace":"test-ns-7bgbq"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"Started","action":"Started","note":"Admitted by clusterQueue cq-lqbf7"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:11.885621365Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-7bgbq","name":"test-preemption-trainjob-xmrql","reconcileID":"0c105433-42f1-4510-bee4-d8d813a84265","job":"test-ns-7bgbq/test-preemption-trainjob-xmrql","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:11.915101572Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-lqbf7","reconcileID":"fda0570e-b02a-451d-9827-d2bf72dd7c72"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:11.915101442Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-7bgbq","name":"lq-vj289","reconcileID":"2ae001de-b429-421f-9083-efc1b19d0316"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:11.97505831Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-vj289","namespace":"test-ns-7bgbq"}}
{"level":"Level(-2)","ts":"2026-11-02T22:20:11.975179658Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-7bgbq","name":"lq-vj289","reconcileID":"444313e6-c08b-4f29-ae3d-1e3f0d41672f"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:11.977733415Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-lqbf7"}}
{"level":"Level(-2)","ts":"2026-11-02T22:20:11.9779059Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-lqbf7","reconcileID":"9ffe449c-48d3-44cf-bba4-15ba3ad25d41"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:12.045330488Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-7bgbq","name":"test-preemption-trainjob-xmrql","reconcileID":"ae7e2778-8af5-4fa6-bbcf-2759473198c2","job":"test-ns-7bgbq/test-preemption-trainjob-xmrql","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:12.202749664Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-7bgbq","name":"test-preemption-trainjob-xmrql","reconcileID":"cb8e50c5-fc86-47df-9c58-50fc4d0446f5","job":"test-ns-7bgbq/test-preemption-trainjob-xmrql","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:12.360035662Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-7bgbq","name":"test-preemption-trainjob-xmrql","reconcileID":"fbbe6017-f891-4295-8836-54acca19bbab","job":"test-ns-7bgbq/test-preemption-trainjob-xmrql","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:12.518025734Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-7bgbq","name":"test-preemption-trainjob-xmrql","reconcileID":"f0ef9309-20e4-4584-8a73-f9e641f5d7e9","job":"test-ns-7bgbq/test-preemption-trainjob-xmrql","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:12.676004254Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-7bgbq","name":"test-preemption-trainjob-xmrql","reconcileID":"23fafaa9-1948-45a8-a5fe-136fb2d24677","job":"test-ns-7bgbq/test-preemption-trainjob-xmrql","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:13.270124329Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-7bgbq","name":"trainjob-test-preemption-trainjob-xmrql-e533e","reconcileID":"2674b91c-67ad-4e05-8f90-2dbdf376fe87"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:14.77961567Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-xmrql-e533e","namespace":"test-ns-7bgbq"},"queue":"lq-vj289","status":"admitted","clusterQueue":"cq-lqbf7"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:14.779687353Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1381","msg":"Workload will not be queued because the workload is not active","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-xmrql-e533e","namespace":"test-ns-7bgbq"},"queue":"lq-vj289","status":"admitted","clusterQueue":"cq-lqbf7"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:14.779686536Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-7bgbq","name":"trainjob-test-preemption-trainjob-xmrql-e533e","reconcileID":"0337691a-1c1f-4c71-a960-60a77146ef65"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:14.779740928Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-7bgbq","name":"trainjob-test-preemption-trainjob-xmrql-e533e","reconcileID":"0337691a-1c1f-4c71-a960-60a77146ef65","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T22:20:14.779739579Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-7bgbq","name":"test-preemption-trainjob-xmrql","reconcileID":"48917409-b1db-4633-9356-d4825c864b8d","job":"test-ns-7bgbq/test-preemption-trainjob-xmrql","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:14.779806411Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-7bgbq","name":"trainjob-test-preemption-trainjob-xmrql-e533e","reconcileID":"56552a63-e356-4ef1-8d58-6dc43af150b1"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:14.89173946Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-xmrql-e533e","namespace":"test-ns-7bgbq"},"queue":"lq-vj289","status":"admitted","clusterQueue":"cq-lqbf7"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:14.891787818Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-7bgbq","name":"trainjob-test-preemption-trainjob-xmrql-e533e","reconcileID":"70d78cb8-be68-4b0f-8b4b-659c1fb011cb"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:14.891811995Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1381","msg":"Workload will not be queued because the workload is not active","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-xmrql-e533e","namespace":"test-ns-7bgbq"},"queue":"lq-vj289","status":"admitted","clusterQueue":"cq-lqbf7"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:14.891829381Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-7bgbq","name":"trainjob-test-preemption-trainjob-xmrql-e533e","reconcileID":"70d78cb8-be68-4b0f-8b4b-659c1fb011cb","isDeleted":false}
{"level":"debug","ts":"2026-11-02T22:20:14.891723432Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-test-preemption-trainjob-xmrql-e533e","namespace":"test-ns-7bgbq"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"EvictedDueToDeactivated","action":"EvictedDueToDeactivated","note":"The workload is deactivated"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:14.89188909Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-7bgbq","name":"trainjob-test-preemption-trainjob-xmrql-e533e","reconcileID":"4259d4fb-1ec3-44ec-993b-a74940fbe4ca"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:14.950254095Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-7bgbq","name":"test-preemption-trainjob-xmrql","reconcileID":"5708ebe2-eede-453f-8929-e31045d9f4b1","job":"test-ns-7bgbq/test-preemption-trainjob-xmrql","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:14.953824068Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-7bgbq","name":"trainjob-test-preemption-trainjob-xmrql-e533e","reconcileID":"36c853b5-b497-4637-b49b-760d5429cc4e"}
{"level":"Level(-3)","ts":"2026-11-02T22:20:15.164587808Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-7bgbq","name":"test-preemption-trainjob-xmrql","reconcileID":"5708ebe2-eede-453f-8929-e31045d9f4b1","error":"error suspending trainjob: Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-preemption-trainjob-xmrql\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:15.164711954Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-7bgbq","name":"test-preemption-trainjob-xmrql","reconcileID":"026e44dd-3348-4f89-bb95-1abc1696b9de","job":"test-ns-7bgbq/test-preemption-trainjob-xmrql","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"info","ts":"2026-11-02T22:20:15.38166881Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-preemption-trainjob-xmrql","namespace":"test-ns-7bgbq"},"namespace":"test-ns-7bgbq","name":"test-preemption-trainjob-xmrql","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"3b923dc5-b707-4686-a87b-aa0da16495c1","replica-role":"leader"}
{"level":"error","ts":"2026-11-02T22:20:15.439147432Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-7bgbq","name":"test-preemption-trainjob-xmrql","reconcileID":"026e44dd-3348-4f89-bb95-1abc1696b9de","error":"jobs are still active","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"debug","ts":"2026-11-02T22:20:15.439232096Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"test-preemption-trainjob-xmrql","namespace":"test-ns-7bgbq"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"Stopped","action":"Stopped","note":"The workload is deactivated"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:15.439398Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-7bgbq","name":"test-preemption-trainjob-xmrql","reconcileID":"1bf1f239-5e16-448e-94c7-a85d908f632d","job":"test-ns-7bgbq/test-preemption-trainjob-xmrql","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-11-02T22:20:15.59672834Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-7bgbq","name":"test-preemption-trainjob-xmrql","reconcileID":"1bf1f239-5e16-448e-94c7-a85d908f632d","error":"jobs are still active","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:15.596850264Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-7bgbq","name":"test-preemption-trainjob-xmrql","reconcileID":"52c8d0b5-9fe9-482d-b0e6-dc9b9dec75fb","job":"test-ns-7bgbq/test-preemption-trainjob-xmrql","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-11-02T22:20:15.75435617Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-7bgbq","name":"test-preemption-trainjob-xmrql","reconcileID":"52c8d0b5-9fe9-482d-b0e6-dc9b9dec75fb","error":"jobs are still active","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:15.754462149Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-7bgbq","name":"test-preemption-trainjob-xmrql","reconcileID":"654e4289-c491-4fcc-8b9b-22560dd673d9","job":"test-ns-7bgbq/test-preemption-trainjob-xmrql","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:15.780322142Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-lqbf7","reconcileID":"d136a786-0e88-4baf-8191-602e6295cd01"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:15.780316926Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-7bgbq","name":"lq-vj289","reconcileID":"4a97825b-c8d5-41e9-b416-0b89fa4f1814"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:15.84006348Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-vj289","namespace":"test-ns-7bgbq"}}
{"level":"Level(-2)","ts":"2026-11-02T22:20:15.840194923Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-7bgbq","name":"lq-vj289","reconcileID":"9d416404-e9c1-4c77-ae13-5e34cce54c53"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:15.843108078Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-lqbf7"}}
{"level":"Level(-2)","ts":"2026-11-02T22:20:15.843266796Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-lqbf7","reconcileID":"a4db731c-1f83-40b0-97b0-235d75132ef5"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:15.850207085Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-lqbf7"}}
{"level":"Level(-2)","ts":"2026-11-02T22:20:15.850243101Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-7bgbq","name":"trainjob-test-preemption-trainjob-xmrql-e533e","reconcileID":"cd01de94-8be2-492c-8439-057daa485f57"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:15.850278143Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-lqbf7","reconcileID":"396d8492-6e26-43cb-9e7c-0012fd870598"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:15.866176224Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:171","msg":"ResourceFlavor update event","replica-role":"leader","resourceFlavor":{"name":"rf-qrb5g"}}
{"level":"Level(-2)","ts":"2026-11-02T22:20:15.866276377Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-qrb5g","reconcileID":"354865d7-3446-4708-9767-bb0b067d65fb"}
{"level":"error","ts":"2026-11-02T22:20:15.912113852Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-7bgbq","name":"test-preemption-trainjob-xmrql","reconcileID":"654e4289-c491-4fcc-8b9b-22560dd673d9","error":"jobs are still active","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:15.912203085Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-7bgbq","name":"test-preemption-trainjob-xmrql","reconcileID":"ce94b0e4-cba5-4568-ad51-051468446b16","job":"test-ns-7bgbq/test-preemption-trainjob-xmrql","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:15.958857726Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:376","msg":"ClusterQueue delete event","replica-role":"leader","clusterQueue":{"name":"cq-lqbf7"}}
{"level":"Level(-2)","ts":"2026-11-02T22:20:15.958891851Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-7bgbq","name":"trainjob-test-preemption-trainjob-xmrql-e533e","reconcileID":"3252af27-6513-4361-b4ac-0e8eb996e491"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:15.958923795Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-7bgbq","name":"lq-vj289","reconcileID":"3fb49bd6-1a59-4c09-b082-49e9f5049776"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:15.958928772Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:385","msg":"Cleared resource metrics for deleted ClusterQueue.","replica-role":"leader","clusterQueue":{"name":"cq-lqbf7"}}
{"level":"Level(-2)","ts":"2026-11-02T22:20:15.959100074Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-qrb5g","reconcileID":"236deaee-903f-40a3-a706-85699235244a"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:16.015487137Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-vj289","namespace":"test-ns-7bgbq"}}
{"level":"Level(-2)","ts":"2026-11-02T22:20:16.015569033Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-7bgbq","name":"lq-vj289","reconcileID":"fa8ae803-0a69-4388-9a5f-c78d5381898b"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:16.066126185Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:159","msg":"ResourceFlavor delete event","replica-role":"leader","resourceFlavor":{"name":"rf-qrb5g"}}
{"level":"error","ts":"2026-11-02T22:20:16.076055445Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-7bgbq","name":"test-preemption-trainjob-xmrql","reconcileID":"ce94b0e4-cba5-4568-ad51-051468446b16","error":"jobs are still active","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:16.076167479Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-7bgbq","name":"test-preemption-trainjob-xmrql","reconcileID":"e4712686-a610-4cec-84c3-de39596af875","job":"test-ns-7bgbq/test-preemption-trainjob-xmrql","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-11-02T22:20:16.129349415Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-7bgbq","name":"test-preemption-trainjob-xmrql","reconcileID":"e4712686-a610-4cec-84c3-de39596af875","job":"test-ns-7bgbq/test-preemption-trainjob-xmrql","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-xmrql\" is forbidden: unable to create new content in namespace test-ns-7bgbq because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-02T22:20:16.129419264Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-7bgbq","name":"test-preemption-trainjob-xmrql","reconcileID":"e4712686-a610-4cec-84c3-de39596af875","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-xmrql\" is forbidden: unable to create new content in namespace test-ns-7bgbq because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:16.157708755Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:141","msg":"ResourceFlavor create event","replica-role":"leader","resourceFlavor":{"name":"rf-gv79f"}}
{"level":"Level(-2)","ts":"2026-11-02T22:20:16.157807951Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-gv79f","reconcileID":"5492305b-138f-4d68-bbea-a9ca45a05600"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:16.274045257Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:351","msg":"ClusterQueue create event","replica-role":"leader","clusterQueue":{"name":"cq-52pvb"}}
{"level":"Level(-2)","ts":"2026-11-02T22:20:16.27427005Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-52pvb","reconcileID":"e9134a0f-3d32-40d1-8968-b9895be81589"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:16.291217985Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:240","msg":"LocalQueue create event","replica-role":"leader","localQueue":{"name":"lq-722jg","namespace":"test-ns-hb254"}}
{"level":"Level(-2)","ts":"2026-11-02T22:20:16.291331582Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-hb254","name":"lq-722jg","reconcileID":"19e4d5b6-7f37-4f74-9a24-7dadc9cc3771"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:16.333724521Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-52pvb"}}
{"level":"Level(-2)","ts":"2026-11-02T22:20:16.333857436Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-52pvb","reconcileID":"e961e990-d0ca-4fac-9584-fb924bdc8d25"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:16.349599789Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-722jg","namespace":"test-ns-hb254"}}
{"level":"Level(-2)","ts":"2026-11-02T22:20:16.349598538Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-hb254","name":"lq-722jg","reconcileID":"c4f1517d-df59-483c-970a-836075023e75"}
{"level":"info","ts":"2026-11-02T22:20:16.355397786Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:117","msg":"Validating create","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-runtimepatches-trainjob-pdk5q","namespace":"test-ns-hb254"},"namespace":"test-ns-hb254","name":"test-runtimepatches-trainjob-pdk5q","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:admin","requestID":"195a33d9-6b36-40ba-a54a-1d67adc15022","replica-role":"leader"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:16.406811042Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-722jg","namespace":"test-ns-hb254"}}
{"level":"Level(-2)","ts":"2026-11-02T22:20:16.406843142Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-hb254","name":"lq-722jg","reconcileID":"407f26e2-815e-4b66-a7f7-19b3fa7f0a2d"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:16.406976176Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-hb254","name":"lq-722jg","reconcileID":"a3f438b1-f6a8-48d9-bddc-0fb286647c58"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:16.411837798Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-hb254","name":"test-runtimepatches-trainjob-pdk5q","reconcileID":"45942b1a-beda-4323-83f5-5eabe48dcd24","job":"test-ns-hb254/test-runtimepatches-trainjob-pdk5q","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:16.727406835Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-7bgbq","name":"test-preemption-trainjob-xmrql","reconcileID":"7ce820ce-920c-4ad5-8e9a-28c149e9d97b","job":"test-ns-7bgbq/test-preemption-trainjob-xmrql","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:16.727443278Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1276","msg":"Workload create event","replica-role":"leader","workload":{"name":"trainjob-test-runtimepatches-trainjob-pdk5q-01925","namespace":"test-ns-hb254"},"queue":"lq-722jg","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:16.727400221Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-hb254","name":"trainjob-test-runtimepatches-trainjob-pdk5q-01925","reconcileID":"81a58d19-b776-4279-bdac-e97aae55c716"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:16.727529902Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-hb254","name":"trainjob-test-runtimepatches-trainjob-pdk5q-01925","reconcileID":"81a58d19-b776-4279-bdac-e97aae55c716","isDeleted":false}
{"level":"debug","ts":"2026-11-02T22:20:16.727402256Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"test-runtimepatches-trainjob-pdk5q","namespace":"test-ns-hb254"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"CreatedWorkload","action":"CreatedWorkload","note":"Created Workload: test-ns-hb254/trainjob-test-runtimepatches-trainjob-pdk5q-01925"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:16.727675938Z","logger":"scheduler","caller":"scheduler/scheduler.go:308","msg":"Obtained heads","replica-role":"leader","schedulingCycle":4,"headCount":1,"waitDuration":5.81249462}
{"level":"Level(-2)","ts":"2026-11-02T22:20:16.727720697Z","logger":"scheduler","caller":"scheduler/scheduler.go:323","msg":"Snapshot taken","replica-role":"leader","schedulingCycle":4,"duration":0.000012806}
{"level":"Level(-2)","ts":"2026-11-02T22:20:16.727722004Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-hb254","name":"trainjob-test-runtimepatches-trainjob-pdk5q-01925","reconcileID":"c9fa2308-a16c-4afb-977d-2bc04b157ead"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:16.727826624Z","logger":"scheduler","caller":"scheduler/scheduler.go:328","msg":"Nomination done","replica-role":"leader","schedulingCycle":4,"entries":1,"inadmissibleEntries":0,"duration":0.000094112}
{"level":"Level(-2)","ts":"2026-11-02T22:20:16.727850733Z","logger":"scheduler","caller":"scheduler/scheduler.go:406","msg":"Attempting to schedule workload","replica-role":"leader","schedulingCycle":4,"workload":{"name":"trainjob-test-runtimepatches-trainjob-pdk5q-01925","namespace":"test-ns-hb254"},"clusterQueue":{"name":"cq-52pvb"}}
{"level":"Level(-2)","ts":"2026-11-02T22:20:16.728021882Z","logger":"scheduler","caller":"scheduler/scheduler.go:1041","msg":"Workload assumed in the cache","replica-role":"leader","schedulingCycle":4,"workload":{"name":"trainjob-test-runtimepatches-trainjob-pdk5q-01925","namespace":"test-ns-hb254"},"clusterQueue":{"name":"cq-52pvb"}}
{"level":"Level(-2)","ts":"2026-11-02T22:20:16.728058026Z","logger":"scheduler","caller":"scheduler/scheduler.go:362","msg":"Workload processing done","replica-role":"leader","schedulingCycle":4,"duration":0.000212234}
{"level":"Level(-2)","ts":"2026-11-02T22:20:16.728078836Z","logger":"scheduler","caller":"scheduler/scheduler.go:297","msg":"Scheduling cycle complete","replica-role":"leader","schedulingCycle":4,"duration":5.812898382}
{"level":"Level(-2)","ts":"2026-11-02T22:20:16.728099939Z","logger":"scheduler","caller":"scheduler/scheduler.go:295","msg":"Scheduling cycle starts","replica-role":"leader","schedulingCycle":5}
{"level":"error","ts":"2026-11-02T22:20:16.780654712Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-7bgbq","name":"test-preemption-trainjob-xmrql","reconcileID":"7ce820ce-920c-4ad5-8e9a-28c149e9d97b","job":"test-ns-7bgbq/test-preemption-trainjob-xmrql","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-xmrql\" is forbidden: unable to create new content in namespace test-ns-7bgbq because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-02T22:20:16.780718368Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-7bgbq","name":"test-preemption-trainjob-xmrql","reconcileID":"7ce820ce-920c-4ad5-8e9a-28c149e9d97b","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-xmrql\" is forbidden: unable to create new content in namespace test-ns-7bgbq because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:16.780802901Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-hb254","name":"test-runtimepatches-trainjob-pdk5q","reconcileID":"6e03da51-038c-4028-8b34-6a8e4a8fc2da","job":"test-ns-hb254/test-runtimepatches-trainjob-pdk5q","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:16.836293751Z","logger":"scheduler","caller":"scheduler/scheduler.go:1000","msg":"Workload successfully admitted and assigned flavors","replica-role":"leader","schedulingCycle":4,"workload":{"name":"trainjob-test-runtimepatches-trainjob-pdk5q-01925","namespace":"test-ns-hb254"},"clusterQueue":{"name":"cq-52pvb"},"assignments":[{"name":"node","flavors":{"cpu":"rf-gv79f","memory":"rf-gv79f"},"resourceUsage":{"cpu":"100m","memory":"128Mi"},"count":1}]}
{"level":"debug","ts":"2026-11-02T22:20:16.836343278Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-test-runtimepatches-trainjob-pdk5q-01925","namespace":"test-ns-hb254"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"QuotaReserved","action":"QuotaReserved","note":"Quota reserved in ClusterQueue cq-52pvb, wait time since queued was 1s"}
{"level":"debug","ts":"2026-11-02T22:20:16.836362608Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-test-runtimepatches-trainjob-pdk5q-01925","namespace":"test-ns-hb254"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"Admitted","action":"Admitted","note":"Admitted by ClusterQueue cq-52pvb, wait time since reservation was 1s"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:16.836539665Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-hb254","name":"trainjob-test-runtimepatches-trainjob-pdk5q-01925","reconcileID":"cb7fe326-6917-435c-a418-e11bbe434d20"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:16.83658142Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-runtimepatches-trainjob-pdk5q-01925","namespace":"test-ns-hb254"},"queue":"lq-722jg","status":"admitted","prevStatus":"pending","clusterQueue":"cq-52pvb"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:16.836598521Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-hb254","name":"trainjob-test-runtimepatches-trainjob-pdk5q-01925","reconcileID":"cb7fe326-6917-435c-a418-e11bbe434d20","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T22:20:16.836802433Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-hb254","name":"trainjob-test-runtimepatches-trainjob-pdk5q-01925","reconcileID":"d932fbcb-9db1-436a-a836-7235b7478d9c"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:16.93817477Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-hb254","name":"test-runtimepatches-trainjob-pdk5q","reconcileID":"b18c9028-98ab-4c5f-b55d-b9453197163e","job":"test-ns-hb254/test-runtimepatches-trainjob-pdk5q","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:17.0952383Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-hb254","name":"test-runtimepatches-trainjob-pdk5q","reconcileID":"b18c9028-98ab-4c5f-b55d-b9453197163e","job":"test-ns-hb254/test-runtimepatches-trainjob-pdk5q","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-3)","ts":"2026-11-02T22:20:17.151544833Z","logger":"trainjob","caller":"jobframework/reconciler.go:594","msg":"Unsuspending job","replica-role":"leader","namespace":"test-ns-hb254","name":"test-runtimepatches-trainjob-pdk5q","reconcileID":"b18c9028-98ab-4c5f-b55d-b9453197163e","job":"test-ns-hb254/test-runtimepatches-trainjob-pdk5q","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-runtimepatches-trainjob-pdk5q\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-3)","ts":"2026-11-02T22:20:17.151626763Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-hb254","name":"test-runtimepatches-trainjob-pdk5q","reconcileID":"b18c9028-98ab-4c5f-b55d-b9453197163e","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-runtimepatches-trainjob-pdk5q\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:17.151727917Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-hb254","name":"test-runtimepatches-trainjob-pdk5q","reconcileID":"43c2cb16-b6a5-4322-a5e2-923a5b5a0586","job":"test-ns-hb254/test-runtimepatches-trainjob-pdk5q","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:17.309676274Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-hb254","name":"test-runtimepatches-trainjob-pdk5q","reconcileID":"43c2cb16-b6a5-4322-a5e2-923a5b5a0586","job":"test-ns-hb254/test-runtimepatches-trainjob-pdk5q","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"info","ts":"2026-11-02T22:20:17.3681949Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-runtimepatches-trainjob-pdk5q","namespace":"test-ns-hb254"},"namespace":"test-ns-hb254","name":"test-runtimepatches-trainjob-pdk5q","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"bee35fa4-8074-4446-8809-2572c7c30a2a","replica-role":"leader"}
{"level":"info","ts":"2026-11-02T22:20:17.485116104Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-runtimepatches-trainjob-pdk5q","namespace":"test-ns-hb254"},"namespace":"test-ns-hb254","name":"test-runtimepatches-trainjob-pdk5q","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"ca07ad82-9d8d-483d-b27e-961672e00e3c","replica-role":"leader"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:17.49288146Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-52pvb"}}
{"level":"Level(-2)","ts":"2026-11-02T22:20:17.492991786Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-52pvb","reconcileID":"c6223096-ba4b-4477-b732-7a278a7d7e6a"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:17.492994989Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-hb254","name":"trainjob-test-runtimepatches-trainjob-pdk5q-01925","reconcileID":"905f781c-9c27-4821-968c-a3e8b3059b05"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:17.508830359Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:171","msg":"ResourceFlavor update event","replica-role":"leader","resourceFlavor":{"name":"rf-gv79f"}}
{"level":"Level(-2)","ts":"2026-11-02T22:20:17.508903795Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-gv79f","reconcileID":"543160f6-4c79-474c-9c7f-5168e9839fe0"}
{"level":"debug","ts":"2026-11-02T22:20:17.542753199Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"test-runtimepatches-trainjob-pdk5q","namespace":"test-ns-hb254"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"Started","action":"Started","note":"Admitted by clusterQueue cq-52pvb"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:17.542798251Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-7bgbq","name":"test-preemption-trainjob-xmrql","reconcileID":"b4e6b93b-a4e8-412a-89cf-4e22d64e2e01","job":"test-ns-7bgbq/test-preemption-trainjob-xmrql","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:17.55136826Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-52pvb"}}
{"level":"Level(-2)","ts":"2026-11-02T22:20:17.551398053Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-hb254","name":"lq-722jg","reconcileID":"b399a0da-d05e-4558-ba36-bceac06a86ee"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:17.551440762Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-52pvb","reconcileID":"d7b58b31-9336-411d-8eba-60efb4eed5b8"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:17.551401382Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-hb254","name":"trainjob-test-runtimepatches-trainjob-pdk5q-01925","reconcileID":"72c40a52-349a-4f4d-a4ed-f70bdb50d148"}
{"level":"error","ts":"2026-11-02T22:20:17.59551177Z","logger":"events","caller":"events/event_broadcaster.go:270","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-hb254","name":"test-runtimepatches-trainjob-pdk5q.18e45a76b829c3e7"},"error":"events.events.k8s.io \"test-runtimepatches-trainjob-pdk5q.18e45a76b829c3e7\" is forbidden: unable to create new content in namespace test-ns-hb254 because it is being terminated","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/workspace/upstream/kueue/src/vendor/k8s.io/client-go/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).attemptRecording\n\t/workspace/upstream/kueue/src/vendor/k8s.io/client-go/tools/events/event_broadcaster.go:211\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).recordToSink.func1\n\t/workspace/upstream/kueue/src/vendor/k8s.io/client-go/tools/events/event_broadcaster.go:200"}
{"level":"error","ts":"2026-11-02T22:20:17.596126396Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-7bgbq","name":"test-preemption-trainjob-xmrql","reconcileID":"b4e6b93b-a4e8-412a-89cf-4e22d64e2e01","job":"test-ns-7bgbq/test-preemption-trainjob-xmrql","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-xmrql\" is forbidden: unable to create new content in namespace test-ns-7bgbq because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-02T22:20:17.596182097Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-7bgbq","name":"test-preemption-trainjob-xmrql","reconcileID":"b4e6b93b-a4e8-412a-89cf-4e22d64e2e01","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-xmrql\" is forbidden: unable to create new content in namespace test-ns-7bgbq because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:17.596275382Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-hb254","name":"test-runtimepatches-trainjob-pdk5q","reconcileID":"60f5746c-53fe-4980-b7c5-96f64712ef3a","job":"test-ns-hb254/test-runtimepatches-trainjob-pdk5q","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:17.608885294Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-722jg","namespace":"test-ns-hb254"}}
{"level":"Level(-2)","ts":"2026-11-02T22:20:17.608994973Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-hb254","name":"lq-722jg","reconcileID":"8a29b542-ce1b-46a7-b88f-61aa69d13998"}
{"level":"error","ts":"2026-11-02T22:20:17.649613537Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-hb254","name":"test-runtimepatches-trainjob-pdk5q","reconcileID":"60f5746c-53fe-4980-b7c5-96f64712ef3a","job":"test-ns-hb254/test-runtimepatches-trainjob-pdk5q","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-runtimepatches-trainjob-pdk5q\" is forbidden: unable to create new content in namespace test-ns-hb254 because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-02T22:20:17.64967819Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-hb254","name":"test-runtimepatches-trainjob-pdk5q","reconcileID":"60f5746c-53fe-4980-b7c5-96f64712ef3a","error":"workloads.kueue.x-k8s.io \"test-runtimepatches-trainjob-pdk5q\" is forbidden: unable to create new content in namespace test-ns-hb254 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:17.654989809Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-hb254","name":"test-runtimepatches-trainjob-pdk5q","reconcileID":"9e2bf877-42da-4440-91ad-8f8f26b66582","job":"test-ns-hb254/test-runtimepatches-trainjob-pdk5q","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-11-02T22:20:17.708679105Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-hb254","name":"test-runtimepatches-trainjob-pdk5q","reconcileID":"9e2bf877-42da-4440-91ad-8f8f26b66582","job":"test-ns-hb254/test-runtimepatches-trainjob-pdk5q","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-runtimepatches-trainjob-pdk5q\" is forbidden: unable to create new content in namespace test-ns-hb254 because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-02T22:20:17.708746061Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-hb254","name":"test-runtimepatches-trainjob-pdk5q","reconcileID":"9e2bf877-42da-4440-91ad-8f8f26b66582","error":"workloads.kueue.x-k8s.io \"test-runtimepatches-trainjob-pdk5q\" is forbidden: unable to create new content in namespace test-ns-hb254 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:17.719070365Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-hb254","name":"test-runtimepatches-trainjob-pdk5q","reconcileID":"396e2cbb-2eba-43ea-904c-0617a30cb495","job":"test-ns-hb254/test-runtimepatches-trainjob-pdk5q","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-11-02T22:20:17.772867667Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-hb254","name":"test-runtimepatches-trainjob-pdk5q","reconcileID":"396e2cbb-2eba-43ea-904c-0617a30cb495","job":"test-ns-hb254/test-runtimepatches-trainjob-pdk5q","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-runtimepatches-trainjob-pdk5q\" is forbidden: unable to create new content in namespace test-ns-hb254 because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-02T22:20:17.772933903Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-hb254","name":"test-runtimepatches-trainjob-pdk5q","reconcileID":"396e2cbb-2eba-43ea-904c-0617a30cb495","error":"workloads.kueue.x-k8s.io \"test-runtimepatches-trainjob-pdk5q\" is forbidden: unable to create new content in namespace test-ns-hb254 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:17.773105391Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-hb254","name":"test-runtimepatches-trainjob-pdk5q","reconcileID":"b5e601b1-db4d-4089-822d-8c7ac943d7f2","job":"test-ns-hb254/test-runtimepatches-trainjob-pdk5q","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-11-02T22:20:17.82641921Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-hb254","name":"test-runtimepatches-trainjob-pdk5q","reconcileID":"b5e601b1-db4d-4089-822d-8c7ac943d7f2","job":"test-ns-hb254/test-runtimepatches-trainjob-pdk5q","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-runtimepatches-trainjob-pdk5q\" is forbidden: unable to create new content in namespace test-ns-hb254 because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-02T22:20:17.826489202Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-hb254","name":"test-runtimepatches-trainjob-pdk5q","reconcileID":"b5e601b1-db4d-4089-822d-8c7ac943d7f2","error":"workloads.kueue.x-k8s.io \"test-runtimepatches-trainjob-pdk5q\" is forbidden: unable to create new content in namespace test-ns-hb254 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:17.843556902Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-hb254","name":"test-runtimepatches-trainjob-pdk5q","reconcileID":"910d2590-86ea-4bd6-9f00-bc939be1b3d2","job":"test-ns-hb254/test-runtimepatches-trainjob-pdk5q","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-11-02T22:20:17.897058735Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-hb254","name":"test-runtimepatches-trainjob-pdk5q","reconcileID":"910d2590-86ea-4bd6-9f00-bc939be1b3d2","job":"test-ns-hb254/test-runtimepatches-trainjob-pdk5q","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-runtimepatches-trainjob-pdk5q\" is forbidden: unable to create new content in namespace test-ns-hb254 because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-02T22:20:17.897126418Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-hb254","name":"test-runtimepatches-trainjob-pdk5q","reconcileID":"910d2590-86ea-4bd6-9f00-bc939be1b3d2","error":"workloads.kueue.x-k8s.io \"test-runtimepatches-trainjob-pdk5q\" is forbidden: unable to create new content in namespace test-ns-hb254 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:17.91328203Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-hb254","name":"test-runtimepatches-trainjob-pdk5q","reconcileID":"5da4acec-35cd-4f19-b384-9d91f6490790","job":"test-ns-hb254/test-runtimepatches-trainjob-pdk5q","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-11-02T22:20:17.966647862Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-hb254","name":"test-runtimepatches-trainjob-pdk5q","reconcileID":"5da4acec-35cd-4f19-b384-9d91f6490790","job":"test-ns-hb254/test-runtimepatches-trainjob-pdk5q","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-runtimepatches-trainjob-pdk5q\" is forbidden: unable to create new content in namespace test-ns-hb254 because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-02T22:20:17.966703085Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-hb254","name":"test-runtimepatches-trainjob-pdk5q","reconcileID":"5da4acec-35cd-4f19-b384-9d91f6490790","error":"workloads.kueue.x-k8s.io \"test-runtimepatches-trainjob-pdk5q\" is forbidden: unable to create new content in namespace test-ns-hb254 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T22:20:17.975195488Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-hb254","name":"test-runtimepatches-trainjob-pdk5q","reconcileID":"0ed29519-f3a1-4ea8-897c-8fe3d482c3ed","job":"test-ns-hb254/test-runtimepatches-trainjob-pdk5q","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}