{"level":"Level(-2)","ts":"2026-11-02T10:09:41.700042059Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-29hzv","name":"e2e-test-flux","reconcileID":"dd3686ed-6117-48ab-b0b6-ef27ec94668f","job":"e2e-29hzv/e2e-test-flux","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-29hzv"}
{"level":"Level(-2)","ts":"2026-11-02T10:09:46.463335519Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-29hzv","name":"e2e-test-flux","reconcileID":"be539961-bf14-45ff-bdf4-6e9953ec73b0","job":"e2e-29hzv/e2e-test-flux","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-29hzv"}
{"level":"Level(-2)","ts":"2026-11-02T10:09:46.732589583Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-29hzv","name":"e2e-test-flux","reconcileID":"21e4df30-2599-444f-b472-920fc01cd9f8","job":"e2e-29hzv/e2e-test-flux","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-29hzv"}
{"level":"Level(-2)","ts":"2026-11-02T10:09:47.470820671Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-29hzv","name":"e2e-test-flux","reconcileID":"e6c030e6-7afc-4f71-aed8-65a7a7736b8f","job":"e2e-29hzv/e2e-test-flux","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-29hzv"}
{"level":"Level(-2)","ts":"2026-11-02T10:09:47.535007914Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-29hzv","name":"e2e-test-flux","reconcileID":"c1ddcc96-e1f6-4d7a-8e1c-632be0d37f2a","job":"e2e-29hzv/e2e-test-flux","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-29hzv"}
{"level":"Level(-2)","ts":"2026-11-02T10:09:47.689994667Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-29hzv","name":"e2e-test-flux","reconcileID":"8cc46beb-56b8-406d-a508-9588989aa913","job":"e2e-29hzv/e2e-test-flux","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-29hzv"}
{"level":"Level(-2)","ts":"2026-11-02T10:09:47.974915235Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-b698h","name":"e2e-test","reconcileID":"7d5744de-33ee-4e83-b98c-b1f5d820f218","job":"e2e-b698h/e2e-test","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-b698h"}
{"level":"Level(-2)","ts":"2026-11-02T10:09:48.170187028Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-j8b7s","name":"e2e-update-time","reconcileID":"c51923c8-4021-4817-a7f8-5f1f9bd010ac","job":"e2e-j8b7s/e2e-update-time","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-j8b7s"}
{"level":"Level(-2)","ts":"2026-11-02T10:09:48.435956687Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-j8b7s","name":"e2e-update-time","reconcileID":"348a5c78-36f3-475b-99ff-10d2841cacef","job":"e2e-j8b7s/e2e-update-time","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-j8b7s"}
{"level":"Level(-2)","ts":"2026-11-02T10:09:48.550404452Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-j8b7s","name":"e2e-update-time","reconcileID":"c319a1cd-e346-4279-8879-a953b3de2127","job":"e2e-j8b7s/e2e-update-time","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-j8b7s"}
{"level":"Level(-2)","ts":"2026-11-02T10:09:48.663316537Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-j8b7s","name":"e2e-update-time","reconcileID":"57c4184f-43de-4848-9439-203a72321632","job":"e2e-j8b7s/e2e-update-time","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-j8b7s"}
{"level":"Level(-2)","ts":"2026-11-02T10:09:48.780157342Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-j8b7s","name":"e2e-update-time","reconcileID":"d75abb14-d8d2-477f-b7c5-d3bb574dfc24","job":"e2e-j8b7s/e2e-update-time","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-j8b7s"}
{"level":"Level(-2)","ts":"2026-11-02T10:09:48.876980429Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-j8b7s","name":"e2e-update-time","reconcileID":"e027938a-0c3c-4c03-a194-54122b3d480a","job":"e2e-j8b7s/e2e-update-time","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-j8b7s"}
{"level":"Level(-2)","ts":"2026-11-02T10:09:49.329824393Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-j8b7s","name":"e2e-update-time","reconcileID":"090c53d4-39a8-44d6-9389-bf87d9bcfa44","job":"e2e-j8b7s/e2e-update-time","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-j8b7s"}
{"level":"Level(-2)","ts":"2026-11-02T10:09:49.515743768Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-p6ck5","name":"e2e-test-managed-by","reconcileID":"e88ec521-f35e-4113-b952-d88806c3ee42","job":"e2e-p6ck5/e2e-test-managed-by","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-p6ck5"}
{"level":"Level(-2)","ts":"2026-11-02T10:09:49.533035794Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-j8b7s","name":"e2e-update-time","reconcileID":"80d9633a-fdd2-4429-8cf7-2d8ab394916d","job":"e2e-j8b7s/e2e-update-time","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-j8b7s"}
{"level":"Level(-2)","ts":"2026-11-02T10:09:54.404386859Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-j8b7s","name":"e2e-update-time","reconcileID":"c0502143-de3f-4d0d-abbb-fc33417884ce","job":"e2e-j8b7s/e2e-update-time","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-j8b7s"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:04.685723751Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-6c7kg","name":"e2e-deadline-job","reconcileID":"d6d8b4a6-dcb9-43cb-b929-983246c75eae","job":"e2e-6c7kg/e2e-deadline-job","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-6c7kg"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:04.907026078Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-6c7kg","name":"e2e-deadline-job","reconcileID":"905c046c-bce8-44a2-ab8c-54fa679467cb","job":"e2e-6c7kg/e2e-deadline-job","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-6c7kg"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:05.138932452Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-6c7kg","name":"e2e-deadline-job","reconcileID":"4c6d7ba3-af57-4848-aa39-a602cfcb3974","job":"e2e-6c7kg/e2e-deadline-job","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-6c7kg"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:05.250411655Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-6c7kg","name":"e2e-deadline-job","reconcileID":"3d44b422-d13d-4fbf-9462-7483de663b82","job":"e2e-6c7kg/e2e-deadline-job","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-6c7kg"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:05.255196019Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-6c7kg","name":"e2e-deadline-job","reconcileID":"80172abc-ed44-4c0e-8d76-912e66cfb114","job":"e2e-6c7kg/e2e-deadline-job","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-6c7kg"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:05.359915023Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-6c7kg","name":"e2e-deadline-job","reconcileID":"596e4da8-82e9-49f0-a0ab-b086fe555384","job":"e2e-6c7kg/e2e-deadline-job","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-6c7kg"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:08.175444041Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-6c7kg","name":"e2e-deadline-job","reconcileID":"b3850d02-73b6-4761-8524-575edf1558c8","job":"e2e-6c7kg/e2e-deadline-job","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-6c7kg"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:08.285003039Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-6c7kg","name":"e2e-deadline-job","reconcileID":"a71a1cad-baa0-4655-be2d-891a40b00dc1","job":"e2e-6c7kg/e2e-deadline-job","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-6c7kg"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:14.109701208Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-6c7kg","name":"e2e-deadline-job","reconcileID":"d73a1fd5-eebf-45fe-9e23-d02eca9b37fa","job":"e2e-6c7kg/e2e-deadline-job","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-6c7kg"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:14.176643808Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-6c7kg","name":"e2e-deadline-job","reconcileID":"ac27489a-ef76-4749-8b9d-94a5772954bc","job":"e2e-6c7kg/e2e-deadline-job","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-6c7kg"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:14.33694985Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-6c7kg","name":"e2e-deadline-job","reconcileID":"e1546f10-dc52-43ee-bfe7-5ac7de34bac1","job":"e2e-6c7kg/e2e-deadline-job","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-6c7kg"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:14.449930556Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-6c7kg","name":"e2e-deadline-job","reconcileID":"901b52da-642d-4584-9a3e-d59e87226f16","job":"e2e-6c7kg/e2e-deadline-job","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-6c7kg"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:14.507305012Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-6c7kg","name":"e2e-deadline-job","reconcileID":"906d8c42-7eef-41fb-81e7-bc55144e1897","job":"e2e-6c7kg/e2e-deadline-job","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-6c7kg"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:14.661946615Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-2lfvw","name":"e2e-test-runtime-update","reconcileID":"774f99ef-83d9-4a92-9b05-1356262ea07f","job":"e2e-2lfvw/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-2lfvw"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:14.881415165Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-2lfvw","name":"e2e-test-runtime-update","reconcileID":"60376564-2e62-4dab-bb60-d1046820a099","job":"e2e-2lfvw/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-2lfvw"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:15.053450795Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-2lfvw","name":"e2e-test-runtime-update","reconcileID":"bc3dbb3d-820a-4d21-b565-2e5c5b863b14","job":"e2e-2lfvw/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-2lfvw"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:15.162549267Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-2lfvw","name":"e2e-test-runtime-update","reconcileID":"808bb9e6-47f2-40f8-bbf1-d40a82e27dbe","job":"e2e-2lfvw/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-2lfvw"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:15.185417238Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-2lfvw","name":"e2e-test-runtime-update","reconcileID":"9198b124-6f00-474f-b9ab-0f2d2267882b","job":"e2e-2lfvw/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-2lfvw"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:15.272774536Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-2lfvw","name":"e2e-test-runtime-update","reconcileID":"db932ebb-4cbf-4182-b842-b8aa287fed99","job":"e2e-2lfvw/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-2lfvw"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:15.614714697Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-2lfvw","name":"e2e-test-runtime-update","reconcileID":"3bfece24-8645-4c5c-9bb2-c900046d08c6","job":"e2e-2lfvw/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-2lfvw"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:15.723899153Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-2lfvw","name":"e2e-test-runtime-update","reconcileID":"34eb4d1b-5570-4522-8720-ed7adb2de8d3","job":"e2e-2lfvw/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-2lfvw"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:15.833400329Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-2lfvw","name":"e2e-test-runtime-update","reconcileID":"2aa95335-3efd-4a10-b693-3ece7db7a771","job":"e2e-2lfvw/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-2lfvw"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:15.845658236Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-2lfvw","name":"e2e-test-runtime-update","reconcileID":"08cd542c-d93d-407b-a4a9-37b8ac305233","job":"e2e-2lfvw/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-2lfvw"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:15.957268661Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-2lfvw","name":"e2e-test-runtime-update","reconcileID":"5b067d36-3e20-4245-9cef-60879f847b63","job":"e2e-2lfvw/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-2lfvw"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:16.055494537Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-2lfvw","name":"e2e-test-runtime-update","reconcileID":"ea290bb7-1854-454b-982e-f2c1a70e13c6","job":"e2e-2lfvw/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-2lfvw"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:16.400682086Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-2lfvw","name":"e2e-test-runtime-update","reconcileID":"17462155-2ec5-4c6c-9b1b-3da16f456013","job":"e2e-2lfvw/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-2lfvw"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:16.55082363Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-2lfvw","name":"e2e-test-runtime-update","reconcileID":"9fa69492-1090-4948-b7fd-122450821f0b","job":"e2e-2lfvw/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-2lfvw"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:16.660086452Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-2lfvw","name":"e2e-test-runtime-update","reconcileID":"775f8dd1-1625-4ebd-983a-f7a91b82cdae","job":"e2e-2lfvw/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-2lfvw"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:16.773043237Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-2lfvw","name":"e2e-test-runtime-update","reconcileID":"deb7f40d-b32f-43f2-b116-89bddc46fbcc","job":"e2e-2lfvw/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-2lfvw"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:16.832473029Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-2lfvw","name":"e2e-test-runtime-update","reconcileID":"60377bdd-f28f-40e7-b3f7-ac976a66603b","job":"e2e-2lfvw/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-2lfvw"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:16.944001145Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-2lfvw","name":"e2e-test-runtime-update","reconcileID":"5d9ab759-d4ea-441c-b109-6d90f3a712eb","job":"e2e-2lfvw/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-2lfvw"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:16.945284958Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-2lfvw","name":"e2e-test-runtime-update","reconcileID":"e28c4779-929c-4634-8a61-aff60a3113aa","job":"e2e-2lfvw/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-2lfvw"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:17.055791557Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-2lfvw","name":"e2e-test-runtime-update","reconcileID":"42569799-8688-44ff-9ed4-6ce7042d7c2c","job":"e2e-2lfvw/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-2lfvw"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:17.64347925Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-r25gd","name":"e2e-test-runtime-update","reconcileID":"3280d04b-32a8-48ed-8205-f54caa7d1008","job":"e2e-r25gd/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-r25gd"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:17.868808069Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-r25gd","name":"e2e-test-runtime-update","reconcileID":"e9a424da-e67f-41e1-bca0-30d23326a870","job":"e2e-r25gd/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-r25gd"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:18.04312995Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-r25gd","name":"e2e-test-runtime-update","reconcileID":"3efff049-99be-4ff9-9387-1cbaa85ce9af","job":"e2e-r25gd/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-r25gd"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:18.15506199Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-r25gd","name":"e2e-test-runtime-update","reconcileID":"ec6ae2fb-8283-497f-8386-5fc220edccfd","job":"e2e-r25gd/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-r25gd"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:18.170834275Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-r25gd","name":"e2e-test-runtime-update","reconcileID":"97f3be59-ad1f-4a45-bdc9-862a78290e8c","job":"e2e-r25gd/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-r25gd"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:18.26492222Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-r25gd","name":"e2e-test-runtime-update","reconcileID":"6008d6ee-6cc2-4f6a-8a01-3b7962035069","job":"e2e-r25gd/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-r25gd"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:18.595060881Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-r25gd","name":"e2e-test-runtime-update","reconcileID":"7b85de93-5c74-471c-b03b-12280e152352","job":"e2e-r25gd/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-r25gd"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:18.707932511Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-r25gd","name":"e2e-test-runtime-update","reconcileID":"dc82c278-1e41-4239-abba-7f50b94d0c4f","job":"e2e-r25gd/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-r25gd"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:18.819536028Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-r25gd","name":"e2e-test-runtime-update","reconcileID":"1dc71b11-b0bd-4502-ae6b-6a3da2896353","job":"e2e-r25gd/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-r25gd"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:18.825188294Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-r25gd","name":"e2e-test-runtime-update","reconcileID":"42886d99-18fa-4585-81a6-c0fe5aa8c2d2","job":"e2e-r25gd/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-r25gd"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:18.941189096Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-r25gd","name":"e2e-test-runtime-update","reconcileID":"db098c2d-184b-40c8-a994-9c33ea8b2553","job":"e2e-r25gd/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-r25gd"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:19.039167689Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-r25gd","name":"e2e-test-runtime-update","reconcileID":"b1b8bf47-675d-42d8-8284-8f5aaa7dd965","job":"e2e-r25gd/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-r25gd"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:19.362529307Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-r25gd","name":"e2e-test-runtime-update","reconcileID":"1007ab64-3321-432a-98cb-4f2f9397eea7","job":"e2e-r25gd/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-r25gd"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:19.477238563Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-r25gd","name":"e2e-test-runtime-update","reconcileID":"bec559ea-e806-4caf-8fd7-f2068a80d094","job":"e2e-r25gd/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-r25gd"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:19.58695261Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-r25gd","name":"e2e-test-runtime-update","reconcileID":"bfa9d6b2-e0ea-4ffa-8f38-93f5b6396fa2","job":"e2e-r25gd/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-r25gd"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:19.653410089Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-r25gd","name":"e2e-test-runtime-update","reconcileID":"491e67d7-e1a1-487b-bec4-ee39212950ef","job":"e2e-r25gd/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-r25gd"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:19.714127636Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-r25gd","name":"e2e-test-runtime-update","reconcileID":"102c3b94-6b41-41b9-9ee3-4c07bec5caa7","job":"e2e-r25gd/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-r25gd"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:19.783131554Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-r25gd","name":"e2e-test-runtime-update","reconcileID":"b936107e-5bf0-4726-af7a-156bcf4b0877","job":"e2e-r25gd/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-r25gd"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:19.834316219Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-r25gd","name":"e2e-test-runtime-update","reconcileID":"221e2578-447d-4106-ab1f-cae9b7127c45","job":"e2e-r25gd/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-r25gd"}
{"level":"Level(-2)","ts":"2026-11-02T10:10:19.94619531Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-r25gd","name":"e2e-test-runtime-update","reconcileID":"76e52ccc-f9c5-404e-a433-a39b1f1d9bd0","job":"e2e-r25gd/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-r25gd"}
{"level":"Level(-2)","ts":"2026-11-02T10:11:15.156850284Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ghnnr","name":"test-dra-trainjob-n26r2","reconcileID":"135d74ff-5394-4f15-a8fc-96a5eb26af58","job":"test-ns-ghnnr/test-dra-trainjob-n26r2","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ghnnr"}
{"level":"Level(-2)","ts":"2026-11-02T10:11:15.38479101Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ghnnr","name":"test-dra-trainjob-n26r2","reconcileID":"8a0a2181-aa32-42eb-913e-5489f3a291fc","job":"test-ns-ghnnr/test-dra-trainjob-n26r2","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ghnnr"}
{"level":"Level(-2)","ts":"2026-11-02T10:11:15.615649022Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ghnnr","name":"test-dra-trainjob-n26r2","reconcileID":"f648b432-39ef-4ff2-b574-71b1909d0b42","job":"test-ns-ghnnr/test-dra-trainjob-n26r2","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ghnnr"}
{"level":"Level(-2)","ts":"2026-11-02T10:11:15.72391213Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ghnnr","name":"test-dra-trainjob-n26r2","reconcileID":"17ce03d4-1cc4-44ea-b06a-dfb57f777b0c","job":"test-ns-ghnnr/test-dra-trainjob-n26r2","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ghnnr"}
{"level":"Level(-2)","ts":"2026-11-02T10:11:15.80702387Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ghnnr","name":"test-dra-trainjob-n26r2","reconcileID":"e332f1a9-9377-4ba7-aaa3-a86d32405d4d","job":"test-ns-ghnnr/test-dra-trainjob-n26r2","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ghnnr"}
{"level":"Level(-2)","ts":"2026-11-02T10:11:15.918286857Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ghnnr","name":"test-dra-trainjob-n26r2","reconcileID":"eae78ed7-d07c-4513-a8f2-b925ce69aaaf","job":"test-ns-ghnnr/test-dra-trainjob-n26r2","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ghnnr"}
{"level":"Level(-2)","ts":"2026-11-02T10:11:19.752438225Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-s6mgj","name":"test-trainjob-v6zq8","reconcileID":"2a96419c-1289-4cff-80de-359c2214bfbc","job":"test-ns-s6mgj/test-trainjob-v6zq8","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-s6mgj"}
{"level":"Level(-2)","ts":"2026-11-02T10:11:19.972992919Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-s6mgj","name":"test-trainjob-v6zq8","reconcileID":"2e1a2cf8-6a76-4350-b12f-df65b2161141","job":"test-ns-s6mgj/test-trainjob-v6zq8","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-s6mgj"}
{"level":"Level(-2)","ts":"2026-11-02T10:11:20.199432614Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-s6mgj","name":"test-trainjob-v6zq8","reconcileID":"9909b953-2032-43f8-82b4-742bce3d9219","job":"test-ns-s6mgj/test-trainjob-v6zq8","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-s6mgj"}
{"level":"Level(-2)","ts":"2026-11-02T10:11:20.311086783Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-s6mgj","name":"test-trainjob-v6zq8","reconcileID":"93736a9c-6888-428c-b739-bc14bd6768f2","job":"test-ns-s6mgj/test-trainjob-v6zq8","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-s6mgj"}
{"level":"Level(-2)","ts":"2026-11-02T10:11:20.312981116Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-s6mgj","name":"test-trainjob-v6zq8","reconcileID":"84461cb5-1e6e-4274-a17d-f5f4134dc5a1","job":"test-ns-s6mgj/test-trainjob-v6zq8","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-s6mgj"}
{"level":"Level(-2)","ts":"2026-11-02T10:11:20.421640208Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-s6mgj","name":"test-trainjob-v6zq8","reconcileID":"c31e9566-9c81-4d61-9ba9-d3728f453479","job":"test-ns-s6mgj/test-trainjob-v6zq8","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-s6mgj"}
{"level":"Level(-2)","ts":"2026-11-02T10:11:22.374845413Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ghnnr","name":"test-dra-trainjob-n26r2","reconcileID":"137a5187-e5c7-4acf-899f-7f05adecaa4c","job":"test-ns-ghnnr/test-dra-trainjob-n26r2","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ghnnr"}
{"level":"Level(-2)","ts":"2026-11-02T10:11:22.48642987Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ghnnr","name":"test-dra-trainjob-n26r2","reconcileID":"291d0e36-4a04-4560-83b5-736f7d05e240","job":"test-ns-ghnnr/test-dra-trainjob-n26r2","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ghnnr"}
{"level":"Level(-2)","ts":"2026-11-02T10:11:22.500320322Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-s6mgj","name":"test-trainjob-v6zq8","reconcileID":"be5e85d6-206a-4863-8c50-71cda0ef0ed2","job":"test-ns-s6mgj/test-trainjob-v6zq8","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-s6mgj"}
{"level":"Level(-2)","ts":"2026-11-02T10:11:22.649870693Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-s6mgj","name":"test-trainjob-v6zq8","reconcileID":"878e3b3e-f1dd-4bc1-b073-64dbe6169cbf","job":"test-ns-s6mgj/test-trainjob-v6zq8","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-s6mgj"}
{"level":"Level(-2)","ts":"2026-11-02T10:11:26.518205218Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-s6mgj","name":"test-trainjob-v6zq8","reconcileID":"25702603-0594-4266-87a0-264b5234fab8","job":"test-ns-s6mgj/test-trainjob-v6zq8","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-s6mgj"}
{"level":"Level(-2)","ts":"2026-11-02T10:11:26.628296152Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-s6mgj","name":"test-trainjob-v6zq8","reconcileID":"b59a4029-6119-4d59-bf01-b477a31c0b1e","job":"test-ns-s6mgj/test-trainjob-v6zq8","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-s6mgj"}
{"level":"Level(-2)","ts":"2026-11-02T10:11:27.527430711Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-s6mgj","name":"test-trainjob-v6zq8","reconcileID":"bb6e304a-dadd-48a7-83c5-50cd888c05bb","job":"test-ns-s6mgj/test-trainjob-v6zq8","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-s6mgj"}
{"level":"Level(-2)","ts":"2026-11-02T10:11:27.58704241Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-s6mgj","name":"test-trainjob-v6zq8","reconcileID":"c4fa970b-77f5-47ea-b869-d6c59b539530","job":"test-ns-s6mgj/test-trainjob-v6zq8","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-s6mgj"}
{"level":"Level(-2)","ts":"2026-11-02T10:11:27.640852404Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-s6mgj","name":"test-trainjob-v6zq8","reconcileID":"db1b911d-cdf8-4c78-997a-ae132ec300c4","job":"test-ns-s6mgj/test-trainjob-v6zq8","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-s6mgj"}
{"level":"Level(-2)","ts":"2026-11-02T10:11:27.75303866Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-s6mgj","name":"test-trainjob-v6zq8","reconcileID":"a9ef250f-2f5a-4329-a980-d6fd46bde712","job":"test-ns-s6mgj/test-trainjob-v6zq8","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-s6mgj"}
{"level":"Level(-2)","ts":"2026-11-02T10:11:28.091445986Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wvx9k","name":"test-trainjob-rxx7d","reconcileID":"9754d4ef-e9df-4cdd-aab1-046c8426c2f3","job":"test-ns-wvx9k/test-trainjob-rxx7d","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wvx9k"}
{"level":"Level(-2)","ts":"2026-11-02T10:11:28.312889422Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wvx9k","name":"test-trainjob-rxx7d","reconcileID":"abbbe4e5-a100-48f0-9226-fdb7171234cc","job":"test-ns-wvx9k/test-trainjob-rxx7d","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wvx9k"}
{"level":"Level(-2)","ts":"2026-11-02T10:11:28.540558979Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wvx9k","name":"test-trainjob-rxx7d","reconcileID":"de00b6ca-7d31-44c9-9bd3-f20fa15c81d0","job":"test-ns-wvx9k/test-trainjob-rxx7d","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wvx9k"}
{"level":"Level(-2)","ts":"2026-11-02T10:11:28.61972715Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wvx9k","name":"test-trainjob-rxx7d","reconcileID":"8aa459aa-25f0-4b79-ab95-fa20425e8b01","job":"test-ns-wvx9k/test-trainjob-rxx7d","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wvx9k"}
{"level":"Level(-2)","ts":"2026-11-02T10:11:28.649130093Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wvx9k","name":"test-trainjob-rxx7d","reconcileID":"ff7abb59-5369-41c6-9e81-49702587e311","job":"test-ns-wvx9k/test-trainjob-rxx7d","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wvx9k"}
{"level":"Level(-2)","ts":"2026-11-02T10:11:28.757780603Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wvx9k","name":"test-trainjob-rxx7d","reconcileID":"0ff4fa08-bfbe-443a-82e3-8d16ae6c8fc5","job":"test-ns-wvx9k/test-trainjob-rxx7d","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wvx9k"}
{"level":"Level(-2)","ts":"2026-11-02T10:16:40.194016962Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wvx9k","name":"test-trainjob-rxx7d","reconcileID":"9bd6c648-7c26-4fb0-b2b4-2fbe0e853179","job":"test-ns-wvx9k/test-trainjob-rxx7d","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wvx9k"}
{"level":"Level(-2)","ts":"2026-11-02T10:16:40.305907496Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wvx9k","name":"test-trainjob-rxx7d","reconcileID":"db1d23de-aaa0-4072-9ff2-862542024a24","job":"test-ns-wvx9k/test-trainjob-rxx7d","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wvx9k"}
{"level":"Level(-2)","ts":"2026-11-02T10:16:45.212915947Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wvx9k","name":"test-trainjob-rxx7d","reconcileID":"f723bec6-a1f3-4ebb-9bec-333c06cb5717","job":"test-ns-wvx9k/test-trainjob-rxx7d","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wvx9k"}
{"level":"Level(-2)","ts":"2026-11-02T10:16:45.322253205Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wvx9k","name":"test-trainjob-rxx7d","reconcileID":"7aac2b91-09a5-4ee7-9b3e-f0028ef95487","job":"test-ns-wvx9k/test-trainjob-rxx7d","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wvx9k"}
{"level":"Level(-2)","ts":"2026-11-02T10:16:46.221729286Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wvx9k","name":"test-trainjob-rxx7d","reconcileID":"7f39b658-b14d-42ad-9c39-356497dff762","job":"test-ns-wvx9k/test-trainjob-rxx7d","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wvx9k"}
{"level":"Level(-2)","ts":"2026-11-02T10:16:46.285043905Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wvx9k","name":"test-trainjob-rxx7d","reconcileID":"a7fde2bd-a148-4d2f-a832-687cb22c7e18","job":"test-ns-wvx9k/test-trainjob-rxx7d","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wvx9k"}
{"level":"Level(-2)","ts":"2026-11-02T10:16:46.332968222Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wvx9k","name":"test-trainjob-rxx7d","reconcileID":"567a23dd-8e2e-4f2b-8d39-5fe56a1ad97d","job":"test-ns-wvx9k/test-trainjob-rxx7d","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wvx9k"}
{"level":"Level(-2)","ts":"2026-11-02T10:16:46.443994115Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wvx9k","name":"test-trainjob-rxx7d","reconcileID":"cfdd7ee7-26cd-4f23-ad45-bff75e75da5a","job":"test-ns-wvx9k/test-trainjob-rxx7d","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wvx9k"}
{"level":"Level(-2)","ts":"2026-11-02T10:16:47.028135409Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-w5c5j","name":"test-trainjob-qb6pw","reconcileID":"89785a75-cbae-4fc1-89e1-4e59022956ce","job":"test-ns-w5c5j/test-trainjob-qb6pw","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-w5c5j"}
{"level":"Level(-2)","ts":"2026-11-02T10:16:47.474980424Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-w5c5j","name":"test-trainjob-qb6pw","reconcileID":"761a3a38-5347-4363-8278-94cb663b9110","job":"test-ns-w5c5j/test-trainjob-qb6pw","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-w5c5j"}
{"level":"Level(-2)","ts":"2026-11-02T10:16:47.70232148Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-w5c5j","name":"test-trainjob-qb6pw","reconcileID":"aa7261d8-c08e-422e-b0a6-f164e9a5a241","job":"test-ns-w5c5j/test-trainjob-qb6pw","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-w5c5j"}
{"level":"Level(-2)","ts":"2026-11-02T10:16:47.810793317Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-w5c5j","name":"test-trainjob-qb6pw","reconcileID":"e4bb3154-bee3-467d-8c3b-ab342bc15ee9","job":"test-ns-w5c5j/test-trainjob-qb6pw","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-w5c5j"}
{"level":"Level(-2)","ts":"2026-11-02T10:16:48.009994136Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-w5c5j","name":"test-trainjob-qb6pw","reconcileID":"8fbc4201-d71a-4548-b0d8-aa97eaf2ce45","job":"test-ns-w5c5j/test-trainjob-qb6pw","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-w5c5j"}
{"level":"Level(-2)","ts":"2026-11-02T10:16:48.120002271Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-w5c5j","name":"test-trainjob-qb6pw","reconcileID":"29f70dd7-426d-4fb9-8dca-2bc4e434f197","job":"test-ns-w5c5j/test-trainjob-qb6pw","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-w5c5j"}
{"level":"Level(-2)","ts":"2026-11-02T10:18:02.638429153Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-w5c5j","name":"test-trainjob-qb6pw","reconcileID":"5fdb637f-975b-4640-85c7-7cf6ff839207","job":"test-ns-w5c5j/test-trainjob-qb6pw","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-w5c5j"}
{"level":"Level(-2)","ts":"2026-11-02T10:18:02.748950182Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-w5c5j","name":"test-trainjob-qb6pw","reconcileID":"3ccd7541-cff9-43a4-8a0e-f84ad1f65c1d","job":"test-ns-w5c5j/test-trainjob-qb6pw","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-w5c5j"}
{"level":"Level(-2)","ts":"2026-11-02T10:18:04.650284857Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-w5c5j","name":"test-trainjob-qb6pw","reconcileID":"44392886-53e6-4190-9c7b-a99784b9119d","job":"test-ns-w5c5j/test-trainjob-qb6pw","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-w5c5j"}
{"level":"Level(-2)","ts":"2026-11-02T10:18:04.759636032Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-w5c5j","name":"test-trainjob-qb6pw","reconcileID":"18669624-8b2c-4cab-b35c-d0f40d0ae441","job":"test-ns-w5c5j/test-trainjob-qb6pw","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-w5c5j"}
{"level":"Level(-2)","ts":"2026-11-02T10:18:05.657142346Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-w5c5j","name":"test-trainjob-qb6pw","reconcileID":"e58af651-ea00-4a15-97af-3f7d094f27d9","job":"test-ns-w5c5j/test-trainjob-qb6pw","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-w5c5j"}
{"level":"Level(-2)","ts":"2026-11-02T10:18:05.718980599Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-w5c5j","name":"test-trainjob-qb6pw","reconcileID":"ad3cd036-2323-4c44-a266-415cdfa10385","job":"test-ns-w5c5j/test-trainjob-qb6pw","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-w5c5j"}
{"level":"Level(-2)","ts":"2026-11-02T10:18:05.765769713Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-w5c5j","name":"test-trainjob-qb6pw","reconcileID":"dc68b457-f8d7-46ab-9a03-87ffa0b2b465","job":"test-ns-w5c5j/test-trainjob-qb6pw","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-w5c5j"}
{"level":"Level(-2)","ts":"2026-11-02T10:18:05.877831516Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-w5c5j","name":"test-trainjob-qb6pw","reconcileID":"7ccc30fd-5e82-4239-8a13-4e6094cc2acd","job":"test-ns-w5c5j/test-trainjob-qb6pw","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-w5c5j"}
{"level":"Level(-2)","ts":"2026-11-02T10:18:07.126148321Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-q4j2b","name":"test-trainjob-qll5j","reconcileID":"f4173c31-7467-4a85-a624-fc099cadde1c","job":"test-ns-q4j2b/test-trainjob-qll5j","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-q4j2b"}
{"level":"Level(-2)","ts":"2026-11-02T10:18:07.411910076Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-q4j2b","name":"test-trainjob-qll5j","reconcileID":"c42c5d55-8842-401e-8202-475ffcd664cb","job":"test-ns-q4j2b/test-trainjob-qll5j","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-q4j2b"}
{"level":"Level(-2)","ts":"2026-11-02T10:18:07.646563215Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-q4j2b","name":"test-trainjob-qll5j","reconcileID":"9ad9e38e-99c2-4091-af78-ea10e55af761","job":"test-ns-q4j2b/test-trainjob-qll5j","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-q4j2b"}
{"level":"Level(-2)","ts":"2026-11-02T10:18:07.759636951Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-q4j2b","name":"test-trainjob-qll5j","reconcileID":"2f6d7664-91cb-4f51-aa8f-749184ceac54","job":"test-ns-q4j2b/test-trainjob-qll5j","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-q4j2b"}
{"level":"Level(-2)","ts":"2026-11-02T10:18:07.766249354Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-q4j2b","name":"test-trainjob-qll5j","reconcileID":"1c1fb4ce-1ddb-442c-aeef-e78484be9724","job":"test-ns-q4j2b/test-trainjob-qll5j","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-q4j2b"}
{"level":"Level(-2)","ts":"2026-11-02T10:18:07.874086239Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-q4j2b","name":"test-trainjob-qll5j","reconcileID":"99cf1c03-8c9a-47fd-ac17-91d31bc05008","job":"test-ns-q4j2b/test-trainjob-qll5j","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-q4j2b"}
{"level":"Level(-2)","ts":"2026-11-02T10:18:11.936870333Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wvx9k","name":"test-trainjob-rxx7d","reconcileID":"55ddecbc-c68c-4999-af77-1042ac05d4e4","job":"test-ns-wvx9k/test-trainjob-rxx7d","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wvx9k"}
{"level":"Level(-2)","ts":"2026-11-02T10:18:12.088133783Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wvx9k","name":"test-trainjob-rxx7d","reconcileID":"8be97aa1-6953-4de5-b653-1b3c16a77342","job":"test-ns-wvx9k/test-trainjob-rxx7d","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wvx9k"}
{"level":"Level(-2)","ts":"2026-11-02T10:18:12.545870127Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-w5c5j","name":"test-trainjob-qb6pw","reconcileID":"b0705a3f-876c-4d93-a8e1-0a6e89752c28","job":"test-ns-w5c5j/test-trainjob-qb6pw","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-w5c5j"}
{"level":"Level(-2)","ts":"2026-11-02T10:18:12.613610705Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-w5c5j","name":"test-trainjob-qb6pw","reconcileID":"a8db20f6-a91b-4294-bef7-f13d39d5f2dd","job":"test-ns-w5c5j/test-trainjob-qb6pw","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-w5c5j"}
{"level":"Level(-2)","ts":"2026-11-02T10:18:12.70256259Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-w5c5j","name":"test-trainjob-qb6pw","reconcileID":"86ac5302-a6b9-46ef-9c72-b23bfcf1c1eb","job":"test-ns-w5c5j/test-trainjob-qb6pw","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-w5c5j"}
{"level":"Level(-2)","ts":"2026-11-02T10:18:28.787723491Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-q4j2b","name":"test-trainjob-qll5j","reconcileID":"801b792b-59d9-4b54-9866-d0ce9bdd36ec","job":"test-ns-q4j2b/test-trainjob-qll5j","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-q4j2b"}
{"level":"Level(-2)","ts":"2026-11-02T10:18:28.902145696Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-q4j2b","name":"test-trainjob-qll5j","reconcileID":"791b94a9-229f-4b69-b019-f70180efcd2d","job":"test-ns-q4j2b/test-trainjob-qll5j","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-q4j2b"}
{"level":"Level(-2)","ts":"2026-11-02T10:18:33.799582245Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-q4j2b","name":"test-trainjob-qll5j","reconcileID":"1f179385-4a9f-49fb-8abe-063def88e107","job":"test-ns-q4j2b/test-trainjob-qll5j","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-q4j2b"}
{"level":"Level(-2)","ts":"2026-11-02T10:18:33.913604761Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-q4j2b","name":"test-trainjob-qll5j","reconcileID":"9356f1d1-9faf-48b5-bd76-6c01abc67eb3","job":"test-ns-q4j2b/test-trainjob-qll5j","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-q4j2b"}
{"level":"Level(-2)","ts":"2026-11-02T10:18:34.809186128Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-q4j2b","name":"test-trainjob-qll5j","reconcileID":"bad2760a-5b57-456f-bd9f-4c27d196119c","job":"test-ns-q4j2b/test-trainjob-qll5j","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-q4j2b"}
{"level":"Level(-2)","ts":"2026-11-02T10:18:34.922833504Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-q4j2b","name":"test-trainjob-qll5j","reconcileID":"6b53200f-7e7f-467c-9b6c-ae9cf2d12d4e","job":"test-ns-q4j2b/test-trainjob-qll5j","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-q4j2b"}
{"level":"Level(-2)","ts":"2026-11-02T10:18:35.037225072Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-q4j2b","name":"test-trainjob-qll5j","reconcileID":"2f492d74-6b7e-42d7-87f7-e4956527d858","job":"test-ns-q4j2b/test-trainjob-qll5j","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-q4j2b"}
{"level":"Level(-2)","ts":"2026-11-02T10:18:35.151446846Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-q4j2b","name":"test-trainjob-qll5j","reconcileID":"7cf4ef56-3652-456f-a3eb-652be8d1d733","job":"test-ns-q4j2b/test-trainjob-qll5j","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-q4j2b"}
{"level":"Level(-2)","ts":"2026-11-02T10:18:35.158363016Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-q4j2b","name":"test-trainjob-qll5j","reconcileID":"28ec618a-8683-4446-9cfe-f1d6f754b354","job":"test-ns-q4j2b/test-trainjob-qll5j","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-q4j2b"}
{"level":"Level(-2)","ts":"2026-11-02T10:18:35.265588322Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-q4j2b","name":"test-trainjob-qll5j","reconcileID":"65a1414e-cf07-4a50-804a-dbf8d16bf535","job":"test-ns-q4j2b/test-trainjob-qll5j","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-q4j2b"}
{"level":"Level(-2)","ts":"2026-11-02T10:19:11.998679748Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-q4j2b","name":"test-trainjob-qll5j","reconcileID":"7e9ac194-b08c-49d6-9399-d14d3169819c","job":"test-ns-q4j2b/test-trainjob-qll5j","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-q4j2b"}
{"level":"Level(-2)","ts":"2026-11-02T10:19:12.112588702Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-q4j2b","name":"test-trainjob-qll5j","reconcileID":"4dce55cf-4643-4ff5-ace2-63296ad1a623","job":"test-ns-q4j2b/test-trainjob-qll5j","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-q4j2b"}
{"level":"Level(-2)","ts":"2026-11-02T10:19:23.05275981Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-q4j2b","name":"test-trainjob-qll5j","reconcileID":"4df83683-8574-4765-bc31-aa1306ee9235","job":"test-ns-q4j2b/test-trainjob-qll5j","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-q4j2b"}
{"level":"Level(-2)","ts":"2026-11-02T10:19:23.167277316Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-q4j2b","name":"test-trainjob-qll5j","reconcileID":"83ed5eec-0b26-4934-b3dd-427d213b750f","job":"test-ns-q4j2b/test-trainjob-qll5j","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-q4j2b"}
{"level":"Level(-2)","ts":"2026-11-02T10:19:24.062708862Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-q4j2b","name":"test-trainjob-qll5j","reconcileID":"16f1b9d5-20e9-4668-92e0-777065c27776","job":"test-ns-q4j2b/test-trainjob-qll5j","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-q4j2b"}
{"level":"Level(-2)","ts":"2026-11-02T10:19:24.175982237Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-q4j2b","name":"test-trainjob-qll5j","reconcileID":"453dfb98-f238-4621-ab79-aa5f9cb832b6","job":"test-ns-q4j2b/test-trainjob-qll5j","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-q4j2b"}
{"level":"Level(-2)","ts":"2026-11-02T10:19:24.28734648Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-q4j2b","name":"test-trainjob-qll5j","reconcileID":"d7a97802-bb10-41b6-976a-9e2042859009","job":"test-ns-q4j2b/test-trainjob-qll5j","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-q4j2b"}
{"level":"Level(-2)","ts":"2026-11-02T10:19:24.40114777Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-q4j2b","name":"test-trainjob-qll5j","reconcileID":"0ab3f3bb-8605-40ad-8e81-03b48ec3aae8","job":"test-ns-q4j2b/test-trainjob-qll5j","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-q4j2b"}
{"level":"Level(-2)","ts":"2026-11-02T10:19:24.408060954Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-q4j2b","name":"test-trainjob-qll5j","reconcileID":"3929120d-af29-4ccb-b146-cba01f58312e","job":"test-ns-q4j2b/test-trainjob-qll5j","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-q4j2b"}
{"level":"Level(-2)","ts":"2026-11-02T10:19:24.513919652Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-q4j2b","name":"test-trainjob-qll5j","reconcileID":"86432c92-fbbf-4455-aeac-63b6aae84c0b","job":"test-ns-q4j2b/test-trainjob-qll5j","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-q4j2b"}
{"level":"Level(-2)","ts":"2026-11-02T10:19:35.252476607Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-q4j2b","name":"test-trainjob-qll5j","reconcileID":"994c8f60-9910-4df3-93ee-c81ce9411c02","job":"test-ns-q4j2b/test-trainjob-qll5j","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-q4j2b"}
{"level":"Level(-2)","ts":"2026-11-02T10:19:35.363829378Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-q4j2b","name":"test-trainjob-qll5j","reconcileID":"a4742320-4138-4e67-9e7f-9b49a25e50cf","job":"test-ns-q4j2b/test-trainjob-qll5j","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-q4j2b"}
{"level":"Level(-2)","ts":"2026-11-02T10:19:35.375058199Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-q4j2b","name":"test-trainjob-qll5j","reconcileID":"6403cc0b-ea54-43fd-9ca1-af225c82c5e3","job":"test-ns-q4j2b/test-trainjob-qll5j","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-q4j2b"}
{"level":"Level(-2)","ts":"2026-11-02T10:19:35.477910014Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-q4j2b","name":"test-trainjob-qll5j","reconcileID":"5d978b5d-03c4-4039-97e0-70fd5900c46e","job":"test-ns-q4j2b/test-trainjob-qll5j","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-q4j2b"}
{"level":"Level(-2)","ts":"2026-11-02T10:20:45.566989933Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-twjsm","name":"hebf46b48aae","reconcileID":"d1f4c071-8601-42a4-8e40-d30862442a8f","job":"test-ns-twjsm/hebf46b48aae","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-twjsm"}
{"level":"Level(-2)","ts":"2026-11-02T10:20:46.018186868Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-twjsm","name":"hebf46b48aae","reconcileID":"03cf3dcb-5dcd-4ab3-aa07-72bb01a066c6","job":"test-ns-twjsm/hebf46b48aae","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-twjsm"}
{"level":"Level(-2)","ts":"2026-11-02T10:20:46.246886392Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-twjsm","name":"hebf46b48aae","reconcileID":"98d02259-b04b-42c0-b120-d767b2924b23","job":"test-ns-twjsm/hebf46b48aae","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-twjsm"}
{"level":"Level(-2)","ts":"2026-11-02T10:20:46.360582988Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-twjsm","name":"hebf46b48aae","reconcileID":"8bb28e7e-dc39-415a-8ac5-d02786648f8a","job":"test-ns-twjsm/hebf46b48aae","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-twjsm"}
{"level":"Level(-2)","ts":"2026-11-02T10:20:46.448285545Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-twjsm","name":"hebf46b48aae","reconcileID":"5d8cd7e8-1319-4b0a-9ff3-681a07da84e1","job":"test-ns-twjsm/hebf46b48aae","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-twjsm"}
{"level":"Level(-2)","ts":"2026-11-02T10:20:46.562195984Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-twjsm","name":"hebf46b48aae","reconcileID":"e6a88b8e-231e-413f-b84b-e69c76a5334c","job":"test-ns-twjsm/hebf46b48aae","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-twjsm"}
{"level":"Level(-2)","ts":"2026-11-02T10:20:48.604801106Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-twjsm","name":"hebf46b48aae","reconcileID":"49e61e17-6079-4fa6-abe0-fe33e66266a8","job":"test-ns-twjsm/hebf46b48aae","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-twjsm"}
{"level":"Level(-2)","ts":"2026-11-02T10:20:48.717894713Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-twjsm","name":"hebf46b48aae","reconcileID":"c8fc5761-9dc1-4d61-b072-6326fcdff05d","job":"test-ns-twjsm/hebf46b48aae","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-twjsm"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:18.71650882Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-twjsm","name":"hebf46b48aae","reconcileID":"fa40e0df-a4f9-4b27-80bc-5a3070d3149b","job":"test-ns-twjsm/hebf46b48aae","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-twjsm"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:18.829833471Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-twjsm","name":"hebf46b48aae","reconcileID":"1970c3bb-5af6-4d61-be02-f9fca9a2ebf3","job":"test-ns-twjsm/hebf46b48aae","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-twjsm"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:19.724474995Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-twjsm","name":"hebf46b48aae","reconcileID":"2416ad28-fd41-4df2-a82f-cecc920804f9","job":"test-ns-twjsm/hebf46b48aae","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-twjsm"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:19.837331561Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-twjsm","name":"hebf46b48aae","reconcileID":"a16cc562-5e21-416a-b44f-ba9e7dab8fe9","job":"test-ns-twjsm/hebf46b48aae","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-twjsm"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:19.883177077Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-twjsm","name":"hebf46b48aae","reconcileID":"644b35b5-5326-4005-8d42-b935cf153795","job":"test-ns-twjsm/hebf46b48aae","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-twjsm"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:19.949811963Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-twjsm","name":"hebf46b48aae","reconcileID":"9d7b7255-488b-4c5a-81be-8be6737dca06","job":"test-ns-twjsm/hebf46b48aae","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-twjsm"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:23.827705044Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:141","msg":"ResourceFlavor create event","replica-role":"leader","resourceFlavor":{"name":"rf-z99dz"}}
{"level":"Level(-2)","ts":"2026-11-02T10:21:23.827843888Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-z99dz","reconcileID":"b9d54628-16d1-45a8-8768-f24deef0d0eb"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:24.17017123Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:351","msg":"ClusterQueue create event","replica-role":"leader","clusterQueue":{"name":"cq-wpbxf"}}
{"level":"Level(-2)","ts":"2026-11-02T10:21:24.170421817Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-wpbxf","reconcileID":"cae130df-a7a5-4eea-a320-d19ae37a8b08"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:24.188986546Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:240","msg":"LocalQueue create event","replica-role":"leader","localQueue":{"name":"lq-jf6hj","namespace":"test-ns-c2wlf"}}
{"level":"Level(-2)","ts":"2026-11-02T10:21:24.189199948Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-c2wlf","name":"lq-jf6hj","reconcileID":"e008d6d1-7564-4d19-9f99-94021514a542"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:24.228621836Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-wpbxf"}}
{"level":"Level(-2)","ts":"2026-11-02T10:21:24.228824477Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-wpbxf","reconcileID":"5611b507-1a3d-4b0d-8636-c783b609d759"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:24.244039196Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-c2wlf","name":"lq-jf6hj","reconcileID":"57f13bbe-d138-45fa-b967-e999e2fd5e75"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:24.244074217Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-jf6hj","namespace":"test-ns-c2wlf"}}
{"level":"Level(-2)","ts":"2026-11-02T10:21:24.301052909Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-c2wlf","name":"lq-jf6hj","reconcileID":"7df278af-93cb-4da9-b67d-d34cbab1ff9f"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:24.301056146Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-jf6hj","namespace":"test-ns-c2wlf"}}
{"level":"Level(-2)","ts":"2026-11-02T10:21:24.301262765Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-c2wlf","name":"lq-jf6hj","reconcileID":"e0bf10dc-b651-41df-82dc-8108fd00e859"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:25.016308852Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-c2wlf","name":"dw-deployment-glgbw-f86849689-d9pqs","reconcileID":"8f7e27f0-29a8-4102-aaeb-6d4e18b362c4","job":"test-ns-c2wlf/dw-deployment-glgbw-f86849689-d9pqs","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:25.06867019Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1.PriorityClass","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"debug","ts":"2026-11-02T10:21:25.502727579Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"dw-deployment-glgbw-f86849689-d9pqs","namespace":"test-ns-c2wlf"},"kind":"Pod","apiVersion":"v1","type":"Normal","reason":"CreatedWorkload","action":"CreatedWorkload","note":"Created Workload: test-ns-c2wlf/pod-dw-deployment-glgbw-f86849689-d9pqs-e433c"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:25.552543224Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1276","msg":"Workload create event","replica-role":"leader","workload":{"name":"pod-dw-deployment-glgbw-f86849689-d9pqs-e433c","namespace":"test-ns-c2wlf"},"queue":"lq-jf6hj","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:25.5526384Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-c2wlf","name":"dw-deployment-glgbw-f86849689-d9pqs","reconcileID":"6c3d308e-b0f6-4899-b253-823028f22312","job":"test-ns-c2wlf/dw-deployment-glgbw-f86849689-d9pqs","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:25.552643116Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-c2wlf","name":"pod-dw-deployment-glgbw-f86849689-d9pqs-e433c","reconcileID":"0345b3d9-96e3-4958-adc3-1243fd36d16a"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:25.552704526Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-c2wlf","name":"pod-dw-deployment-glgbw-f86849689-d9pqs-e433c","reconcileID":"0345b3d9-96e3-4958-adc3-1243fd36d16a","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T10:21:25.55282583Z","logger":"scheduler","caller":"scheduler/scheduler.go:308","msg":"Obtained heads","replica-role":"follower","schedulingCycle":1,"headCount":1,"waitDuration":1371.822410826}
{"level":"Level(-2)","ts":"2026-11-02T10:21:25.552865739Z","logger":"scheduler","caller":"scheduler/scheduler.go:323","msg":"Snapshot taken","replica-role":"follower","schedulingCycle":1,"duration":0.000009451}
{"level":"Level(-2)","ts":"2026-11-02T10:21:25.552898559Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-c2wlf","name":"pod-dw-deployment-glgbw-f86849689-d9pqs-e433c","reconcileID":"460360b9-6227-4b1c-8c09-55257d133503"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:25.552966185Z","logger":"scheduler","caller":"scheduler/scheduler.go:328","msg":"Nomination done","replica-role":"follower","schedulingCycle":1,"entries":1,"inadmissibleEntries":0,"duration":0.000091683}
{"level":"Level(-2)","ts":"2026-11-02T10:21:25.552984789Z","logger":"scheduler","caller":"scheduler/scheduler.go:406","msg":"Attempting to schedule workload","replica-role":"follower","schedulingCycle":1,"workload":{"name":"pod-dw-deployment-glgbw-f86849689-d9pqs-e433c","namespace":"test-ns-c2wlf"},"clusterQueue":{"name":"cq-wpbxf"}}
{"level":"Level(-2)","ts":"2026-11-02T10:21:25.553069664Z","logger":"scheduler","caller":"scheduler/scheduler.go:1041","msg":"Workload assumed in the cache","replica-role":"follower","schedulingCycle":1,"workload":{"name":"pod-dw-deployment-glgbw-f86849689-d9pqs-e433c","namespace":"test-ns-c2wlf"},"clusterQueue":{"name":"cq-wpbxf"}}
{"level":"Level(-2)","ts":"2026-11-02T10:21:25.553123238Z","logger":"scheduler","caller":"scheduler/scheduler.go:362","msg":"Workload processing done","replica-role":"follower","schedulingCycle":1,"duration":0.000142106}
{"level":"Level(-2)","ts":"2026-11-02T10:21:25.553140876Z","logger":"scheduler","caller":"scheduler/scheduler.go:297","msg":"Scheduling cycle complete","replica-role":"follower","schedulingCycle":1,"duration":1371.822726769}
{"level":"Level(-2)","ts":"2026-11-02T10:21:25.553156721Z","logger":"scheduler","caller":"scheduler/scheduler.go:295","msg":"Scheduling cycle starts","replica-role":"leader","schedulingCycle":2}
{"level":"Level(-2)","ts":"2026-11-02T10:21:25.667708081Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"pod-dw-deployment-glgbw-f86849689-d9pqs-e433c","namespace":"test-ns-c2wlf"},"queue":"lq-jf6hj","status":"admitted","prevStatus":"pending","clusterQueue":"cq-wpbxf"}
{"level":"debug","ts":"2026-11-02T10:21:25.667715949Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"pod-dw-deployment-glgbw-f86849689-d9pqs-e433c","namespace":"test-ns-c2wlf"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"QuotaReserved","action":"QuotaReserved","note":"Quota reserved in ClusterQueue cq-wpbxf, wait time since queued was 1s"}
{"level":"debug","ts":"2026-11-02T10:21:25.667758417Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"pod-dw-deployment-glgbw-f86849689-d9pqs-e433c","namespace":"test-ns-c2wlf"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"Admitted","action":"Admitted","note":"Admitted by ClusterQueue cq-wpbxf, wait time since reservation was 1s"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:25.667746562Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-c2wlf","name":"pod-dw-deployment-glgbw-f86849689-d9pqs-e433c","reconcileID":"e0c92ef5-529c-44d8-a6e7-c5d98acb23f5"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:25.667767339Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-c2wlf","name":"dw-deployment-glgbw-f86849689-d9pqs","reconcileID":"2d121d82-0c73-4773-a81a-469f758c2772","job":"test-ns-c2wlf/dw-deployment-glgbw-f86849689-d9pqs","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:25.667797609Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-c2wlf","name":"pod-dw-deployment-glgbw-f86849689-d9pqs-e433c","reconcileID":"e0c92ef5-529c-44d8-a6e7-c5d98acb23f5","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T10:21:25.667693781Z","logger":"scheduler","caller":"scheduler/scheduler.go:1000","msg":"Workload successfully admitted and assigned flavors","replica-role":"follower","schedulingCycle":1,"workload":{"name":"pod-dw-deployment-glgbw-f86849689-d9pqs-e433c","namespace":"test-ns-c2wlf"},"clusterQueue":{"name":"cq-wpbxf"},"assignments":[{"name":"main","flavors":{"cpu":"rf-z99dz","memory":"rf-z99dz"},"resourceUsage":{"cpu":"1","memory":"3Gi"},"count":1}]}
{"level":"Level(-2)","ts":"2026-11-02T10:21:25.667967767Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-c2wlf","name":"pod-dw-deployment-glgbw-f86849689-d9pqs-e433c","reconcileID":"e7dc5b72-daa4-4576-b398-d6f0ab1957dc"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:25.667956816Z","logger":"v1_pod","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-c2wlf","name":"dw-deployment-glgbw-f86849689-d9pqs","reconcileID":"2d121d82-0c73-4773-a81a-469f758c2772","job":"test-ns-c2wlf/dw-deployment-glgbw-f86849689-d9pqs","gvk":"/v1, Kind=Pod"}
{"level":"debug","ts":"2026-11-02T10:21:25.783222281Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"dw-deployment-glgbw-f86849689-d9pqs","namespace":"test-ns-c2wlf"},"kind":"Pod","apiVersion":"v1","type":"Normal","reason":"Started","action":"Started","note":"Admitted by clusterQueue cq-wpbxf"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:25.783787913Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-c2wlf","name":"dw-deployment-glgbw-f86849689-d9pqs","reconcileID":"0e8c9249-b86b-47b3-972f-afd2739d5459","job":"test-ns-c2wlf/dw-deployment-glgbw-f86849689-d9pqs","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:25.79091191Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-c2wlf","name":"dw-deployment-glgbw-f86849689-d9pqs","reconcileID":"1ef74331-942d-4e45-adf4-7a976a67b32b","job":"test-ns-c2wlf/dw-deployment-glgbw-f86849689-d9pqs","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:25.852417351Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-c2wlf","name":"dw-deployment-glgbw-f86849689-d9pqs","reconcileID":"4b027425-46a8-479d-a2e0-470b46b9a3e8","job":"test-ns-c2wlf/dw-deployment-glgbw-f86849689-d9pqs","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:25.907351741Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-c2wlf","name":"dw-deployment-glgbw-f86849689-d9pqs","reconcileID":"b71e1eac-66d7-4ab0-94fc-b164f6f744a6","job":"test-ns-c2wlf/dw-deployment-glgbw-f86849689-d9pqs","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:26.553492148Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-c2wlf","name":"lq-jf6hj","reconcileID":"e4cee073-c2d9-4746-affd-402d2b524a4d"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:26.553495541Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-wpbxf","reconcileID":"04b8089c-3b59-4e39-99f4-369c6f02d059"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:26.610288005Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-jf6hj","namespace":"test-ns-c2wlf"}}
{"level":"Level(-2)","ts":"2026-11-02T10:21:26.610414679Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-c2wlf","name":"lq-jf6hj","reconcileID":"bcb49927-82a0-498f-b897-112e177056e4"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:26.633343792Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-wpbxf"}}
{"level":"Level(-2)","ts":"2026-11-02T10:21:26.633503873Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-wpbxf","reconcileID":"8748fbca-3a85-4ac6-a9ff-2f2bc4725f7a"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:26.772754746Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-c2wlf","name":"dw-deployment-glgbw-f86849689-d9pqs","reconcileID":"5f2840ab-7a0f-4d73-8e6e-e41c3832a99e","job":"test-ns-c2wlf/dw-deployment-glgbw-f86849689-d9pqs","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:26.783993077Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-c2wlf","name":"pod-dw-deployment-glgbw-f86849689-d9pqs-e433c","reconcileID":"cb517ff8-39b4-4355-9d14-b3ee6f17fa72"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:28.242980838Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-c2wlf","name":"dw-deployment-glgbw-f86849689-d9pqs","reconcileID":"7275d16a-47e3-4d5d-86df-22f773330695","job":"test-ns-c2wlf/dw-deployment-glgbw-f86849689-d9pqs","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:29.243467139Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-c2wlf","name":"pod-dw-deployment-glgbw-f86849689-d9pqs-e433c","reconcileID":"23696456-2dcd-448f-9b6c-24a0924b710c"}
{"level":"info","ts":"2026-11-02T10:21:39.841728742Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:117","msg":"Validating create","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"bd753176113b","namespace":"test-ns-c2wlf"},"namespace":"test-ns-c2wlf","name":"bd753176113b","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:opendatahub:trainer-e2e","requestID":"4dd4d42e-8dcd-4668-b642-74b76e82a479","replica-role":"leader"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:39.902146024Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-c2wlf","name":"bd753176113b","reconcileID":"6d36a0e4-1a98-4b2c-ac54-01914aad3a58","job":"test-ns-c2wlf/bd753176113b","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"debug","ts":"2026-11-02T10:21:40.223227335Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"bd753176113b","namespace":"test-ns-c2wlf"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"CreatedWorkload","action":"CreatedWorkload","note":"Created Workload: test-ns-c2wlf/trainjob-bd753176113b-25a93"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:40.223282271Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-c2wlf","name":"bd753176113b","reconcileID":"bb3174ba-7981-4317-a167-f485fe9772af","job":"test-ns-c2wlf/bd753176113b","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:40.223511614Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1276","msg":"Workload create event","replica-role":"leader","workload":{"name":"trainjob-bd753176113b-25a93","namespace":"test-ns-c2wlf"},"queue":"lq-jf6hj","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:40.223521056Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-c2wlf","name":"trainjob-bd753176113b-25a93","reconcileID":"9c560874-c106-4c44-a1eb-fd8cbe2d613c"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:40.223574044Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-c2wlf","name":"trainjob-bd753176113b-25a93","reconcileID":"9c560874-c106-4c44-a1eb-fd8cbe2d613c","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T10:21:40.223712809Z","logger":"scheduler","caller":"scheduler/scheduler.go:308","msg":"Obtained heads","replica-role":"leader","schedulingCycle":2,"headCount":1,"waitDuration":14.670555244}
{"level":"Level(-2)","ts":"2026-11-02T10:21:40.223732619Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-c2wlf","name":"trainjob-bd753176113b-25a93","reconcileID":"3c74726a-6395-4b88-8347-3479a1c71582"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:40.223753867Z","logger":"scheduler","caller":"scheduler/scheduler.go:323","msg":"Snapshot taken","replica-role":"leader","schedulingCycle":2,"duration":0.000013704}
{"level":"Level(-2)","ts":"2026-11-02T10:21:40.22382412Z","logger":"scheduler","caller":"scheduler/scheduler.go:328","msg":"Nomination done","replica-role":"leader","schedulingCycle":2,"entries":1,"inadmissibleEntries":0,"duration":0.000061972}
{"level":"Level(-2)","ts":"2026-11-02T10:21:40.223856558Z","logger":"scheduler","caller":"scheduler/scheduler.go:406","msg":"Attempting to schedule workload","replica-role":"leader","schedulingCycle":2,"workload":{"name":"trainjob-bd753176113b-25a93","namespace":"test-ns-c2wlf"},"clusterQueue":{"name":"cq-wpbxf"}}
{"level":"Level(-2)","ts":"2026-11-02T10:21:40.223947697Z","logger":"scheduler","caller":"scheduler/scheduler.go:1041","msg":"Workload assumed in the cache","replica-role":"leader","schedulingCycle":2,"workload":{"name":"trainjob-bd753176113b-25a93","namespace":"test-ns-c2wlf"},"clusterQueue":{"name":"cq-wpbxf"}}
{"level":"Level(-2)","ts":"2026-11-02T10:21:40.223981097Z","logger":"scheduler","caller":"scheduler/scheduler.go:362","msg":"Workload processing done","replica-role":"leader","schedulingCycle":2,"duration":0.000131165}
{"level":"Level(-2)","ts":"2026-11-02T10:21:40.223994744Z","logger":"scheduler","caller":"scheduler/scheduler.go:297","msg":"Scheduling cycle complete","replica-role":"leader","schedulingCycle":2,"duration":14.670837822}
{"level":"Level(-2)","ts":"2026-11-02T10:21:40.224008643Z","logger":"scheduler","caller":"scheduler/scheduler.go:295","msg":"Scheduling cycle starts","replica-role":"leader","schedulingCycle":3}
{"level":"Level(-2)","ts":"2026-11-02T10:21:40.336342989Z","logger":"scheduler","caller":"scheduler/scheduler.go:1000","msg":"Workload successfully admitted and assigned flavors","replica-role":"leader","schedulingCycle":2,"workload":{"name":"trainjob-bd753176113b-25a93","namespace":"test-ns-c2wlf"},"clusterQueue":{"name":"cq-wpbxf"},"assignments":[{"name":"node","flavors":{"cpu":"rf-z99dz","memory":"rf-z99dz"},"resourceUsage":{"cpu":"4","memory":"16Gi"},"count":2}]}
{"level":"debug","ts":"2026-11-02T10:21:40.3364002Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-bd753176113b-25a93","namespace":"test-ns-c2wlf"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"QuotaReserved","action":"QuotaReserved","note":"Quota reserved in ClusterQueue cq-wpbxf, wait time since queued was 0s"}
{"level":"debug","ts":"2026-11-02T10:21:40.336432742Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-bd753176113b-25a93","namespace":"test-ns-c2wlf"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"Admitted","action":"Admitted","note":"Admitted by ClusterQueue cq-wpbxf, wait time since reservation was 0s"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:40.336604838Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-bd753176113b-25a93","namespace":"test-ns-c2wlf"},"queue":"lq-jf6hj","status":"admitted","prevStatus":"pending","clusterQueue":"cq-wpbxf"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:40.33666035Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-c2wlf","name":"trainjob-bd753176113b-25a93","reconcileID":"8d279453-9331-414e-ab56-3bd4e6e1c640"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:40.336710461Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-c2wlf","name":"trainjob-bd753176113b-25a93","reconcileID":"8d279453-9331-414e-ab56-3bd4e6e1c640","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T10:21:40.336901495Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-c2wlf","name":"trainjob-bd753176113b-25a93","reconcileID":"9f205165-df1f-40f4-a9fb-db2f8d522059"}
{"level":"error","ts":"2026-11-02T10:21:40.544523514Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-c2wlf","name":"bd753176113b","reconcileID":"bb3174ba-7981-4317-a167-f485fe9772af","error":"workloads.kueue.x-k8s.io \"trainjob-bd753176113b-25a93\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:40.54466205Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-c2wlf","name":"bd753176113b","reconcileID":"a32bc63c-5ebc-4bec-9428-5f48e0f4dc24","job":"test-ns-c2wlf/bd753176113b","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:40.704132492Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-c2wlf","name":"bd753176113b","reconcileID":"a32bc63c-5ebc-4bec-9428-5f48e0f4dc24","job":"test-ns-c2wlf/bd753176113b","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-3)","ts":"2026-11-02T10:21:40.814625267Z","logger":"trainjob","caller":"jobframework/reconciler.go:594","msg":"Unsuspending job","replica-role":"leader","namespace":"test-ns-c2wlf","name":"bd753176113b","reconcileID":"a32bc63c-5ebc-4bec-9428-5f48e0f4dc24","job":"test-ns-c2wlf/bd753176113b","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"bd753176113b\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-3)","ts":"2026-11-02T10:21:40.81470527Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-c2wlf","name":"bd753176113b","reconcileID":"a32bc63c-5ebc-4bec-9428-5f48e0f4dc24","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"bd753176113b\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:40.814820813Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-c2wlf","name":"bd753176113b","reconcileID":"d4f494ee-50f1-40be-9506-af7162695f43","job":"test-ns-c2wlf/bd753176113b","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:40.974047377Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-c2wlf","name":"bd753176113b","reconcileID":"d4f494ee-50f1-40be-9506-af7162695f43","job":"test-ns-c2wlf/bd753176113b","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"info","ts":"2026-11-02T10:21:41.086213733Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"bd753176113b","namespace":"test-ns-c2wlf"},"namespace":"test-ns-c2wlf","name":"bd753176113b","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"7c10b2f0-22fc-4db6-ab3d-2cb2bc7f3373","replica-role":"leader"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:41.223963059Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-c2wlf","name":"lq-jf6hj","reconcileID":"8b39490c-7bba-46d1-8fb3-e0947b415f7e"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:41.22396314Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-wpbxf","reconcileID":"91b0baa0-35bb-48e7-a437-e7323192f0ef"}
{"level":"info","ts":"2026-11-02T10:21:41.257482774Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"bd753176113b","namespace":"test-ns-c2wlf"},"namespace":"test-ns-c2wlf","name":"bd753176113b","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"1c64aea7-ad01-455e-9527-2549a1520858","replica-role":"leader"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:41.279170824Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-jf6hj","namespace":"test-ns-c2wlf"}}
{"level":"Level(-2)","ts":"2026-11-02T10:21:41.27931769Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-c2wlf","name":"lq-jf6hj","reconcileID":"15e530bc-9f31-4ff0-b3db-ed0f57b62b28"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:41.282395393Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-wpbxf"}}
{"level":"Level(-2)","ts":"2026-11-02T10:21:41.282555856Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-wpbxf","reconcileID":"d88ded07-b5de-4269-aa11-89981eb73ff0"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:41.314065022Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-c2wlf","name":"bd753176113b","reconcileID":"7d9c692e-cf31-48f6-8555-d0caecf2a92c","job":"test-ns-c2wlf/bd753176113b","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"debug","ts":"2026-11-02T10:21:41.31405427Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"bd753176113b","namespace":"test-ns-c2wlf"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"Started","action":"Started","note":"Admitted by clusterQueue cq-wpbxf"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:41.474380775Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-c2wlf","name":"bd753176113b","reconcileID":"7d9c692e-cf31-48f6-8555-d0caecf2a92c","job":"test-ns-c2wlf/bd753176113b","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-3)","ts":"2026-11-02T10:21:41.582726366Z","logger":"trainjob","caller":"jobframework/reconciler.go:594","msg":"Unsuspending job","replica-role":"leader","namespace":"test-ns-c2wlf","name":"bd753176113b","reconcileID":"7d9c692e-cf31-48f6-8555-d0caecf2a92c","job":"test-ns-c2wlf/bd753176113b","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"bd753176113b\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-3)","ts":"2026-11-02T10:21:41.582806306Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-c2wlf","name":"bd753176113b","reconcileID":"7d9c692e-cf31-48f6-8555-d0caecf2a92c","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"bd753176113b\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:41.582931736Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-c2wlf","name":"bd753176113b","reconcileID":"04bcc91a-366d-49b4-a2ad-0030db5c725c","job":"test-ns-c2wlf/bd753176113b","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:41.741928226Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-c2wlf","name":"bd753176113b","reconcileID":"965ecf01-2537-4cb1-ab11-2a8bb198dd6b","job":"test-ns-c2wlf/bd753176113b","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:41.901214662Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-c2wlf","name":"bd753176113b","reconcileID":"ea05badb-7327-407c-93cd-40ec3f1e8bc2","job":"test-ns-c2wlf/bd753176113b","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:42.062464075Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-c2wlf","name":"bd753176113b","reconcileID":"e4fd74de-4a92-4f1d-85cc-92db01ee7889","job":"test-ns-c2wlf/bd753176113b","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:42.423108374Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-c2wlf","name":"bd753176113b","reconcileID":"846be7ad-2ce9-415c-a9ea-1f1ffa50c783","job":"test-ns-c2wlf/bd753176113b","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:42.583336467Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-c2wlf","name":"bd753176113b","reconcileID":"ddd43964-8b39-489f-b165-43f428273579","job":"test-ns-c2wlf/bd753176113b","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:43.177960263Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-c2wlf","name":"trainjob-bd753176113b-25a93","reconcileID":"51f73598-aecc-409a-a981-70b0d8b686bd"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:44.269332049Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-c2wlf","name":"trainjob-bd753176113b-25a93","reconcileID":"e1c8c32e-3d4b-480d-8488-1f5626aa5970"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:44.339691122Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-c2wlf","name":"bd753176113b","reconcileID":"b277f05a-2816-4c5c-867e-6903b44971f7","job":"test-ns-c2wlf/bd753176113b","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T10:21:44.498872552Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-c2wlf","name":"bd753176113b","reconcileID":"8baf7297-687f-4072-aff9-efdf7cdfec63","job":"test-ns-c2wlf/bd753176113b","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:15.035752951Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-c2wlf","name":"trainjob-bd753176113b-25a93","reconcileID":"06586f8b-4928-4a67-941e-3125250a436c"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:15.056555279Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-c2wlf","name":"bd753176113b","reconcileID":"b0edcc15-d023-45f1-b970-c35f02d6cb08","job":"test-ns-c2wlf/bd753176113b","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:15.217031528Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-c2wlf","name":"bd753176113b","reconcileID":"b309dbb7-ccb5-48f1-9bf8-de3be4f3b612","job":"test-ns-c2wlf/bd753176113b","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:16.065652163Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-c2wlf","name":"bd753176113b","reconcileID":"b810f6b3-bc5e-4f83-8c72-368ca529dcb8","job":"test-ns-c2wlf/bd753176113b","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:16.126355748Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-c2wlf","name":"trainjob-bd753176113b-25a93","reconcileID":"22fc74a7-e69d-4bd5-bfe6-09fe59756538"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:16.225592131Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-c2wlf","name":"bd753176113b","reconcileID":"b43c114f-d0b2-4961-b891-f02c480b4b55","job":"test-ns-c2wlf/bd753176113b","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:16.385256806Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-c2wlf","name":"bd753176113b","reconcileID":"a0578197-ca80-4627-9a1b-97e8d5628737","job":"test-ns-c2wlf/bd753176113b","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"debug","ts":"2026-11-02T10:22:16.657652653Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"bd753176113b","namespace":"test-ns-c2wlf"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"FinishedWorkload","action":"FinishedWorkload","note":"Workload 'test-ns-c2wlf/trainjob-bd753176113b-25a93' is declared finished"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:16.657797392Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-bd753176113b-25a93","namespace":"test-ns-c2wlf"},"queue":"lq-jf6hj","status":"finished","prevStatus":"admitted","clusterQueue":"cq-wpbxf"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:16.657832988Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-c2wlf","name":"trainjob-bd753176113b-25a93","reconcileID":"d072eae5-8960-4d0b-84fb-7c5ddb8b89cf"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:16.65788314Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-c2wlf","name":"trainjob-bd753176113b-25a93","reconcileID":"d072eae5-8960-4d0b-84fb-7c5ddb8b89cf","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T10:22:16.658006661Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-c2wlf","name":"trainjob-bd753176113b-25a93","reconcileID":"28ee8919-5711-48e9-9d54-22b1d3857ecd"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:16.658014807Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-c2wlf","name":"bd753176113b","reconcileID":"beb42400-b45d-4c6e-aef9-39f7729ec31c","job":"test-ns-c2wlf/bd753176113b","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"debug","ts":"2026-11-02T10:22:16.836225957Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"bd753176113b","namespace":"test-ns-c2wlf"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"FinishedWorkload","action":"FinishedWorkload","note":"Workload 'test-ns-c2wlf/trainjob-bd753176113b-25a93' is declared finished"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:16.949530535Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-bd753176113b-25a93","namespace":"test-ns-c2wlf"},"queue":"lq-jf6hj","status":"finished","clusterQueue":"cq-wpbxf"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:16.94953878Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-c2wlf","name":"trainjob-bd753176113b-25a93","reconcileID":"b65790a5-175c-4340-84ee-bd37536718a3"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:16.949685477Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-c2wlf","name":"trainjob-bd753176113b-25a93","reconcileID":"b65790a5-175c-4340-84ee-bd37536718a3","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T10:22:16.949691143Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-c2wlf","name":"trainjob-bd753176113b-25a93","reconcileID":"bd5144ff-0fac-4a20-8e70-70fd3629422c"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:16.949699378Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-c2wlf","name":"bd753176113b","reconcileID":"da66c7d8-ade1-4e8e-b628-7c13928eac8a","job":"test-ns-c2wlf/bd753176113b","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"debug","ts":"2026-11-02T10:22:17.112086933Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"bd753176113b","namespace":"test-ns-c2wlf"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"FinishedWorkload","action":"FinishedWorkload","note":"Workload 'test-ns-c2wlf/trainjob-bd753176113b-25a93' is declared finished"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:17.658212081Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-c2wlf","name":"lq-jf6hj","reconcileID":"43ce52e2-113e-4624-950c-86ab337334eb"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:17.658214205Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-wpbxf","reconcileID":"359a4108-d850-42a2-b38b-008e49cbe403"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:17.716915997Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-jf6hj","namespace":"test-ns-c2wlf"}}
{"level":"Level(-2)","ts":"2026-11-02T10:22:17.717039156Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-c2wlf","name":"lq-jf6hj","reconcileID":"ecd37a1f-0a4d-4916-8e14-3d7c8ccfcdd5"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:17.718151982Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-wpbxf"}}
{"level":"Level(-2)","ts":"2026-11-02T10:22:17.718290364Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-wpbxf","reconcileID":"2d2dcd5f-cc45-415b-97f7-8fa377bf6a59"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:18.062393445Z","logger":"trainjob","caller":"workload/workload.go:1810","msg":"Workload is orphaned; finishing to release quota","replica-role":"leader","namespace":"test-ns-c2wlf","name":"bd753176113b","reconcileID":"4750c413-8bbc-4c91-a0b9-dda0df4b2b61","job":"test-ns-c2wlf/bd753176113b","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:18.112211308Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-c2wlf","name":"trainjob-bd753176113b-25a93","reconcileID":"aab2a7e5-7698-43c5-bce1-4508e5d6352d"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:18.11226382Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-c2wlf","name":"trainjob-bd753176113b-25a93","reconcileID":"aab2a7e5-7698-43c5-bce1-4508e5d6352d","isDeleted":true}
{"level":"Level(-2)","ts":"2026-11-02T10:22:18.112254662Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1312","msg":"Workload delete event","replica-role":"leader","workload":{"name":"trainjob-bd753176113b-25a93","namespace":"test-ns-c2wlf"},"queue":"lq-jf6hj","status":"finished"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:19.079469822Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-wpbxf"}}
{"level":"Level(-2)","ts":"2026-11-02T10:22:19.079560977Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-wpbxf","reconcileID":"fc4f7fa9-e2e1-42fa-8626-ef510248c0e5"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:19.079639893Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-c2wlf","name":"pod-dw-deployment-glgbw-f86849689-d9pqs-e433c","reconcileID":"5d4d9e37-cf82-4c08-94c4-2ab3a030aed5"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:19.096867397Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:171","msg":"ResourceFlavor update event","replica-role":"leader","resourceFlavor":{"name":"rf-z99dz"}}
{"level":"Level(-2)","ts":"2026-11-02T10:22:19.096988018Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-z99dz","reconcileID":"fbb61198-0d4b-4890-bd56-d43bf9fc9b6a"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:19.112920598Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-c2wlf","name":"lq-jf6hj","reconcileID":"526d4e20-1aa4-4466-b484-b0ba4d02f5f3"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:19.13725196Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-wpbxf"}}
{"level":"Level(-2)","ts":"2026-11-02T10:22:19.137305332Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-wpbxf","reconcileID":"b4b4385a-4b76-4e77-b024-fe62c820e4d7"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:19.137363166Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-c2wlf","name":"lq-jf6hj","reconcileID":"b6443ac4-a865-467e-8a47-4e307823e069"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:19.137434814Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-c2wlf","name":"pod-dw-deployment-glgbw-f86849689-d9pqs-e433c","reconcileID":"2f2a0b16-73d2-48c4-833c-26a6d92f0c2e"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:19.193984435Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-jf6hj","namespace":"test-ns-c2wlf"}}
{"level":"Level(-2)","ts":"2026-11-02T10:22:19.194254057Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-c2wlf","name":"lq-jf6hj","reconcileID":"3fb65ead-8629-4ffe-82c6-94493901b545"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:19.454400162Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:141","msg":"ResourceFlavor create event","replica-role":"leader","resourceFlavor":{"name":"rf-gs7vs"}}
{"level":"Level(-2)","ts":"2026-11-02T10:22:19.45452582Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-gs7vs","reconcileID":"8324efa9-7cf0-4f2a-b8b7-af3d960eb515"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:19.575929111Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:351","msg":"ClusterQueue create event","replica-role":"leader","clusterQueue":{"name":"cq-tb92j"}}
{"level":"Level(-2)","ts":"2026-11-02T10:22:19.576156638Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-tb92j","reconcileID":"7b6c0b65-59f7-4cb4-8729-a652a707ba09"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:19.593765631Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:240","msg":"LocalQueue create event","replica-role":"leader","localQueue":{"name":"lq-vx4mr","namespace":"test-ns-98gpc"}}
{"level":"Level(-2)","ts":"2026-11-02T10:22:19.593871556Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-98gpc","name":"lq-vx4mr","reconcileID":"4d567a6c-0efe-4d6e-84ec-237a8dc483d5"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:19.633442932Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-tb92j"}}
{"level":"Level(-2)","ts":"2026-11-02T10:22:19.633615499Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-tb92j","reconcileID":"10dd6b5f-864c-4d31-9f13-3fb2a4867d07"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:19.650551808Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-vx4mr","namespace":"test-ns-98gpc"}}
{"level":"Level(-2)","ts":"2026-11-02T10:22:19.650699297Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-98gpc","name":"lq-vx4mr","reconcileID":"a677bd5c-063a-4f38-9e81-c6079d43daf7"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:19.705963649Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-vx4mr","namespace":"test-ns-98gpc"}}
{"level":"Level(-2)","ts":"2026-11-02T10:22:19.706073729Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-98gpc","name":"lq-vx4mr","reconcileID":"cfc4406f-d220-47db-ac2b-0a26a8e2023b"}
{"level":"info","ts":"2026-11-02T10:22:19.713773452Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:117","msg":"Validating create","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-preemption-trainjob-pzmfq","namespace":"test-ns-98gpc"},"namespace":"test-ns-98gpc","name":"test-preemption-trainjob-pzmfq","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:admin","requestID":"629e5475-33d6-4d31-b6ae-f031f37bfb29","replica-role":"leader"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:19.766917456Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-98gpc","name":"test-preemption-trainjob-pzmfq","reconcileID":"b97c50dd-8be2-4561-ab58-0b9015849e8a","job":"test-ns-98gpc/test-preemption-trainjob-pzmfq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:20.059595676Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-c2wlf","name":"pod-dw-deployment-glgbw-f86849689-d9pqs-e433c","reconcileID":"776ac712-a36c-424b-bc6c-a483079a1b84"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:20.083653044Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-98gpc","name":"test-preemption-trainjob-pzmfq","reconcileID":"c38f810c-b2aa-46d0-9866-d9c6657e846a","job":"test-ns-98gpc/test-preemption-trainjob-pzmfq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"debug","ts":"2026-11-02T10:22:20.083675362Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"test-preemption-trainjob-pzmfq","namespace":"test-ns-98gpc"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"CreatedWorkload","action":"CreatedWorkload","note":"Created Workload: test-ns-98gpc/trainjob-test-preemption-trainjob-pzmfq-e726d"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:20.083715642Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-98gpc","name":"trainjob-test-preemption-trainjob-pzmfq-e726d","reconcileID":"553d6c6f-08a1-43d5-9414-4ab25e3119fb"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:20.083647149Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1276","msg":"Workload create event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-pzmfq-e726d","namespace":"test-ns-98gpc"},"queue":"lq-vx4mr","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:20.083762418Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-98gpc","name":"trainjob-test-preemption-trainjob-pzmfq-e726d","reconcileID":"553d6c6f-08a1-43d5-9414-4ab25e3119fb","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T10:22:20.083961921Z","logger":"scheduler","caller":"scheduler/scheduler.go:308","msg":"Obtained heads","replica-role":"leader","schedulingCycle":3,"headCount":1,"waitDuration":39.859952582}
{"level":"Level(-2)","ts":"2026-11-02T10:22:20.083963694Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-98gpc","name":"trainjob-test-preemption-trainjob-pzmfq-e726d","reconcileID":"f855e99f-21f5-4675-adea-1a3f146bb1b6"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:20.083999968Z","logger":"scheduler","caller":"scheduler/scheduler.go:323","msg":"Snapshot taken","replica-role":"leader","schedulingCycle":3,"duration":0.000012216}
{"level":"Level(-2)","ts":"2026-11-02T10:22:20.084074845Z","logger":"scheduler","caller":"scheduler/scheduler.go:328","msg":"Nomination done","replica-role":"leader","schedulingCycle":3,"entries":1,"inadmissibleEntries":0,"duration":0.000064615}
{"level":"Level(-2)","ts":"2026-11-02T10:22:20.084127198Z","logger":"scheduler","caller":"scheduler/scheduler.go:406","msg":"Attempting to schedule workload","replica-role":"leader","schedulingCycle":3,"workload":{"name":"trainjob-test-preemption-trainjob-pzmfq-e726d","namespace":"test-ns-98gpc"},"clusterQueue":{"name":"cq-tb92j"}}
{"level":"Level(-2)","ts":"2026-11-02T10:22:20.084189717Z","logger":"scheduler","caller":"scheduler/scheduler.go:1041","msg":"Workload assumed in the cache","replica-role":"leader","schedulingCycle":3,"workload":{"name":"trainjob-test-preemption-trainjob-pzmfq-e726d","namespace":"test-ns-98gpc"},"clusterQueue":{"name":"cq-tb92j"}}
{"level":"Level(-2)","ts":"2026-11-02T10:22:20.0842129Z","logger":"scheduler","caller":"scheduler/scheduler.go:362","msg":"Workload processing done","replica-role":"leader","schedulingCycle":3,"duration":0.000089374}
{"level":"Level(-2)","ts":"2026-11-02T10:22:20.084226339Z","logger":"scheduler","caller":"scheduler/scheduler.go:297","msg":"Scheduling cycle complete","replica-role":"leader","schedulingCycle":3,"duration":39.860217518}
{"level":"Level(-2)","ts":"2026-11-02T10:22:20.084239909Z","logger":"scheduler","caller":"scheduler/scheduler.go:295","msg":"Scheduling cycle starts","replica-role":"leader","schedulingCycle":4}
{"level":"Level(-2)","ts":"2026-11-02T10:22:20.193731473Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-pzmfq-e726d","namespace":"test-ns-98gpc"},"queue":"lq-vx4mr","status":"admitted","prevStatus":"pending","clusterQueue":"cq-tb92j"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:20.193753732Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-98gpc","name":"trainjob-test-preemption-trainjob-pzmfq-e726d","reconcileID":"6a91caa7-e73f-4886-908f-e84b84f638ab"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:20.193805574Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-98gpc","name":"trainjob-test-preemption-trainjob-pzmfq-e726d","reconcileID":"6a91caa7-e73f-4886-908f-e84b84f638ab","isDeleted":false}
{"level":"debug","ts":"2026-11-02T10:22:20.193809817Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-test-preemption-trainjob-pzmfq-e726d","namespace":"test-ns-98gpc"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"QuotaReserved","action":"QuotaReserved","note":"Quota reserved in ClusterQueue cq-tb92j, wait time since queued was 0s"}
{"level":"debug","ts":"2026-11-02T10:22:20.193834711Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-test-preemption-trainjob-pzmfq-e726d","namespace":"test-ns-98gpc"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"Admitted","action":"Admitted","note":"Admitted by ClusterQueue cq-tb92j, wait time since reservation was 0s"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:20.193797687Z","logger":"scheduler","caller":"scheduler/scheduler.go:1000","msg":"Workload successfully admitted and assigned flavors","replica-role":"leader","schedulingCycle":3,"workload":{"name":"trainjob-test-preemption-trainjob-pzmfq-e726d","namespace":"test-ns-98gpc"},"clusterQueue":{"name":"cq-tb92j"},"assignments":[{"name":"node","count":1}]}
{"level":"Level(-2)","ts":"2026-11-02T10:22:20.193942404Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-98gpc","name":"trainjob-test-preemption-trainjob-pzmfq-e726d","reconcileID":"e5b2e248-752f-4bec-bbd8-8adcdb958752"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:20.240442888Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-98gpc","name":"test-preemption-trainjob-pzmfq","reconcileID":"5b126ecc-0964-410d-99b8-765c14719b0e","job":"test-ns-98gpc/test-preemption-trainjob-pzmfq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:20.398060127Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-98gpc","name":"test-preemption-trainjob-pzmfq","reconcileID":"5b126ecc-0964-410d-99b8-765c14719b0e","job":"test-ns-98gpc/test-preemption-trainjob-pzmfq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-3)","ts":"2026-11-02T10:22:20.504790473Z","logger":"trainjob","caller":"jobframework/reconciler.go:594","msg":"Unsuspending job","replica-role":"leader","namespace":"test-ns-98gpc","name":"test-preemption-trainjob-pzmfq","reconcileID":"5b126ecc-0964-410d-99b8-765c14719b0e","job":"test-ns-98gpc/test-preemption-trainjob-pzmfq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-preemption-trainjob-pzmfq\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-3)","ts":"2026-11-02T10:22:20.504863072Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-98gpc","name":"test-preemption-trainjob-pzmfq","reconcileID":"5b126ecc-0964-410d-99b8-765c14719b0e","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-preemption-trainjob-pzmfq\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:20.504951949Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-98gpc","name":"test-preemption-trainjob-pzmfq","reconcileID":"b31d0361-58fc-4b08-9111-f45892929b7a","job":"test-ns-98gpc/test-preemption-trainjob-pzmfq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:20.661930036Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-98gpc","name":"test-preemption-trainjob-pzmfq","reconcileID":"b31d0361-58fc-4b08-9111-f45892929b7a","job":"test-ns-98gpc/test-preemption-trainjob-pzmfq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-3)","ts":"2026-11-02T10:22:20.768532093Z","logger":"trainjob","caller":"jobframework/reconciler.go:594","msg":"Unsuspending job","replica-role":"leader","namespace":"test-ns-98gpc","name":"test-preemption-trainjob-pzmfq","reconcileID":"b31d0361-58fc-4b08-9111-f45892929b7a","job":"test-ns-98gpc/test-preemption-trainjob-pzmfq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-preemption-trainjob-pzmfq\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-3)","ts":"2026-11-02T10:22:20.76860825Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-98gpc","name":"test-preemption-trainjob-pzmfq","reconcileID":"b31d0361-58fc-4b08-9111-f45892929b7a","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-preemption-trainjob-pzmfq\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:20.768715023Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-98gpc","name":"test-preemption-trainjob-pzmfq","reconcileID":"88fdc7a7-adb4-46c3-bc86-8d0dab57177e","job":"test-ns-98gpc/test-preemption-trainjob-pzmfq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:20.925707544Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-98gpc","name":"test-preemption-trainjob-pzmfq","reconcileID":"88fdc7a7-adb4-46c3-bc86-8d0dab57177e","job":"test-ns-98gpc/test-preemption-trainjob-pzmfq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"info","ts":"2026-11-02T10:22:21.033476219Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-preemption-trainjob-pzmfq","namespace":"test-ns-98gpc"},"namespace":"test-ns-98gpc","name":"test-preemption-trainjob-pzmfq","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"c11a11e0-3b0e-41d1-91fb-77d2ed524d8e","replica-role":"leader"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:21.084367481Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-98gpc","name":"lq-vx4mr","reconcileID":"219f3699-cdb5-4bf5-a1bb-006d61dff283"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:21.084362275Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-tb92j","reconcileID":"0a649037-2b40-49be-a67e-729b64186348"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:21.13980105Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-vx4mr","namespace":"test-ns-98gpc"}}
{"level":"Level(-2)","ts":"2026-11-02T10:22:21.139908037Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-98gpc","name":"lq-vx4mr","reconcileID":"d49a7bde-20e8-447f-adcf-803c18195628"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:21.142922478Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-tb92j"}}
{"level":"Level(-2)","ts":"2026-11-02T10:22:21.143048277Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-tb92j","reconcileID":"b0664a6b-17ad-44ee-8993-7a0476796083"}
{"level":"info","ts":"2026-11-02T10:22:21.196578171Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-preemption-trainjob-pzmfq","namespace":"test-ns-98gpc"},"namespace":"test-ns-98gpc","name":"test-preemption-trainjob-pzmfq","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"fb5547f6-e578-4f6f-8efe-77771a6c46ca","replica-role":"leader"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:21.252734863Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-98gpc","name":"test-preemption-trainjob-pzmfq","reconcileID":"66ed426c-94db-4ac0-af40-4e3822165281","job":"test-ns-98gpc/test-preemption-trainjob-pzmfq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"debug","ts":"2026-11-02T10:22:21.252730688Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"test-preemption-trainjob-pzmfq","namespace":"test-ns-98gpc"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"Started","action":"Started","note":"Admitted by clusterQueue cq-tb92j"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:21.409391486Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-98gpc","name":"test-preemption-trainjob-pzmfq","reconcileID":"25ff0f44-2f22-4133-a887-f07f5723e358","job":"test-ns-98gpc/test-preemption-trainjob-pzmfq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:21.56624122Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-98gpc","name":"test-preemption-trainjob-pzmfq","reconcileID":"4c94f760-1686-4a4d-96e2-f7528e3828ee","job":"test-ns-98gpc/test-preemption-trainjob-pzmfq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:21.723266821Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-98gpc","name":"test-preemption-trainjob-pzmfq","reconcileID":"818d6476-2bf1-4626-a644-2f5fa72e70da","job":"test-ns-98gpc/test-preemption-trainjob-pzmfq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:21.880361983Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-98gpc","name":"test-preemption-trainjob-pzmfq","reconcileID":"723085c3-16f3-420f-af0c-1643319394cf","job":"test-ns-98gpc/test-preemption-trainjob-pzmfq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:22.598817036Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-98gpc","name":"trainjob-test-preemption-trainjob-pzmfq-e726d","reconcileID":"36491fe5-799d-4e3f-ac3f-4f609d1d00bb"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:23.953288336Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-98gpc","name":"trainjob-test-preemption-trainjob-pzmfq-e726d","reconcileID":"695bec35-5586-4da5-b332-bb1415af09e0"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:23.953311721Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-pzmfq-e726d","namespace":"test-ns-98gpc"},"queue":"lq-vx4mr","status":"admitted","clusterQueue":"cq-tb92j"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:23.953347062Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-98gpc","name":"trainjob-test-preemption-trainjob-pzmfq-e726d","reconcileID":"695bec35-5586-4da5-b332-bb1415af09e0","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T10:22:23.953366013Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1381","msg":"Workload will not be queued because the workload is not active","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-pzmfq-e726d","namespace":"test-ns-98gpc"},"queue":"lq-vx4mr","status":"admitted","clusterQueue":"cq-tb92j"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:23.953460684Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-98gpc","name":"trainjob-test-preemption-trainjob-pzmfq-e726d","reconcileID":"8444f715-8945-4afc-b8c3-22e7c5774b57"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:23.953487576Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-98gpc","name":"test-preemption-trainjob-pzmfq","reconcileID":"bed015fb-0cc7-49cc-b02e-5c859c3e1f8e","job":"test-ns-98gpc/test-preemption-trainjob-pzmfq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:23.984054841Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-98gpc","name":"trainjob-test-preemption-trainjob-pzmfq-e726d","reconcileID":"3c05c37b-df45-4ab3-bec2-a78a277bf58f"}
{"level":"debug","ts":"2026-11-02T10:22:24.065886738Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-test-preemption-trainjob-pzmfq-e726d","namespace":"test-ns-98gpc"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"EvictedDueToDeactivated","action":"EvictedDueToDeactivated","note":"The workload is deactivated"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:24.065964973Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-98gpc","name":"trainjob-test-preemption-trainjob-pzmfq-e726d","reconcileID":"17fb68b4-3d14-400e-87ad-b0afc485459c"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:24.0659888Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-pzmfq-e726d","namespace":"test-ns-98gpc"},"queue":"lq-vx4mr","status":"admitted","clusterQueue":"cq-tb92j"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:24.066022973Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-98gpc","name":"trainjob-test-preemption-trainjob-pzmfq-e726d","reconcileID":"17fb68b4-3d14-400e-87ad-b0afc485459c","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T10:22:24.066047339Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1381","msg":"Workload will not be queued because the workload is not active","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-pzmfq-e726d","namespace":"test-ns-98gpc"},"queue":"lq-vx4mr","status":"admitted","clusterQueue":"cq-tb92j"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:24.06618673Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-98gpc","name":"trainjob-test-preemption-trainjob-pzmfq-e726d","reconcileID":"8ca497e4-9dfb-43e1-b10d-62263bcf30a9"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:24.111154645Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-98gpc","name":"test-preemption-trainjob-pzmfq","reconcileID":"ea22aefb-5b11-47cc-bead-0c45ab8dfa5e","job":"test-ns-98gpc/test-preemption-trainjob-pzmfq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-3)","ts":"2026-11-02T10:22:24.374815385Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-98gpc","name":"test-preemption-trainjob-pzmfq","reconcileID":"ea22aefb-5b11-47cc-bead-0c45ab8dfa5e","error":"error suspending trainjob: Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-preemption-trainjob-pzmfq\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:24.374953275Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-98gpc","name":"test-preemption-trainjob-pzmfq","reconcileID":"3bdb20c7-8241-4c74-8ac0-4c3f538be890","job":"test-ns-98gpc/test-preemption-trainjob-pzmfq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"info","ts":"2026-11-02T10:22:24.640212483Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-preemption-trainjob-pzmfq","namespace":"test-ns-98gpc"},"namespace":"test-ns-98gpc","name":"test-preemption-trainjob-pzmfq","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"2f81ca9c-d56a-420f-84a5-9a8ff9da1ec6","replica-role":"leader"}
{"level":"error","ts":"2026-11-02T10:22:24.694353061Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-98gpc","name":"test-preemption-trainjob-pzmfq","reconcileID":"3bdb20c7-8241-4c74-8ac0-4c3f538be890","error":"jobs are still active","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"debug","ts":"2026-11-02T10:22:24.694360724Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"test-preemption-trainjob-pzmfq","namespace":"test-ns-98gpc"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"Stopped","action":"Stopped","note":"The workload is deactivated"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:24.694487609Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-98gpc","name":"test-preemption-trainjob-pzmfq","reconcileID":"c304a1ca-cd25-41a1-abd3-5b60faa0b806","job":"test-ns-98gpc/test-preemption-trainjob-pzmfq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-11-02T10:22:24.852311606Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-98gpc","name":"test-preemption-trainjob-pzmfq","reconcileID":"c304a1ca-cd25-41a1-abd3-5b60faa0b806","error":"jobs are still active","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:24.852463375Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-98gpc","name":"test-preemption-trainjob-pzmfq","reconcileID":"06157988-e20b-4834-936c-2472d19c00bb","job":"test-ns-98gpc/test-preemption-trainjob-pzmfq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:24.953921663Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-tb92j","reconcileID":"40eb3fee-06df-4ce8-9719-6f258cad7977"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:24.95393095Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-98gpc","name":"lq-vx4mr","reconcileID":"cbcc467c-a542-47c4-9c92-0360dd442577"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:25.010942643Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-vx4mr","namespace":"test-ns-98gpc"}}
{"level":"error","ts":"2026-11-02T10:22:25.011273394Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-98gpc","name":"test-preemption-trainjob-pzmfq","reconcileID":"06157988-e20b-4834-936c-2472d19c00bb","error":"jobs are still active","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:25.011300649Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-98gpc","name":"lq-vx4mr","reconcileID":"61894931-a8f0-48fc-9e67-7588048556c9"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:25.011362299Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-98gpc","name":"test-preemption-trainjob-pzmfq","reconcileID":"453ba854-0477-49f8-b62a-d5164e948d92","job":"test-ns-98gpc/test-preemption-trainjob-pzmfq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:25.01221462Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-tb92j"}}
{"level":"Level(-2)","ts":"2026-11-02T10:22:25.01233516Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-tb92j","reconcileID":"cbe078a6-95c5-4e1f-b7e1-65dd7a5957df"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:25.023142884Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-tb92j"}}
{"level":"Level(-2)","ts":"2026-11-02T10:22:25.023202384Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-tb92j","reconcileID":"32a2bde6-6483-4b34-a170-d6f8ef3b6696"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:25.023263631Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-98gpc","name":"trainjob-test-preemption-trainjob-pzmfq-e726d","reconcileID":"10876052-34ab-473c-8e02-b56975dcb87c"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:25.042361477Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:171","msg":"ResourceFlavor update event","replica-role":"leader","resourceFlavor":{"name":"rf-gs7vs"}}
{"level":"Level(-2)","ts":"2026-11-02T10:22:25.042442726Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-gs7vs","reconcileID":"37462ced-eb90-45f4-a41b-e9a36e94e200"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:25.13165231Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-98gpc","name":"trainjob-test-preemption-trainjob-pzmfq-e726d","reconcileID":"6cc1ded8-6670-4287-b265-fbb068089446"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:25.131642097Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:376","msg":"ClusterQueue delete event","replica-role":"leader","clusterQueue":{"name":"cq-tb92j"}}
{"level":"Level(-2)","ts":"2026-11-02T10:22:25.131704414Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-98gpc","name":"lq-vx4mr","reconcileID":"21a654c4-6248-4130-a8bf-969f13b9c74c"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:25.131754547Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:385","msg":"Cleared resource metrics for deleted ClusterQueue.","replica-role":"leader","clusterQueue":{"name":"cq-tb92j"}}
{"level":"Level(-2)","ts":"2026-11-02T10:22:25.131882229Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-gs7vs","reconcileID":"aede5602-f323-4f6d-913d-bdb8d4dbcb4c"}
{"level":"error","ts":"2026-11-02T10:22:25.16865253Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-98gpc","name":"test-preemption-trainjob-pzmfq","reconcileID":"453ba854-0477-49f8-b62a-d5164e948d92","error":"jobs are still active","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:25.168759298Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-98gpc","name":"test-preemption-trainjob-pzmfq","reconcileID":"774cc5f8-20c3-4647-a552-bbcf483159f8","job":"test-ns-98gpc/test-preemption-trainjob-pzmfq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:25.18832229Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-vx4mr","namespace":"test-ns-98gpc"}}
{"level":"Level(-2)","ts":"2026-11-02T10:22:25.188516514Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-98gpc","name":"lq-vx4mr","reconcileID":"389c1756-af79-415f-a222-c72397b0cb08"}
{"level":"error","ts":"2026-11-02T10:22:25.220302055Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-98gpc","name":"test-preemption-trainjob-pzmfq","reconcileID":"774cc5f8-20c3-4647-a552-bbcf483159f8","job":"test-ns-98gpc/test-preemption-trainjob-pzmfq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-pzmfq\" is forbidden: unable to create new content in namespace test-ns-98gpc because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-02T10:22:25.220359947Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-98gpc","name":"test-preemption-trainjob-pzmfq","reconcileID":"774cc5f8-20c3-4647-a552-bbcf483159f8","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-pzmfq\" is forbidden: unable to create new content in namespace test-ns-98gpc because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:25.220434554Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-98gpc","name":"test-preemption-trainjob-pzmfq","reconcileID":"500d7751-fb29-4b88-93bb-a88b997bee81","job":"test-ns-98gpc/test-preemption-trainjob-pzmfq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:25.239762302Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:159","msg":"ResourceFlavor delete event","replica-role":"leader","resourceFlavor":{"name":"rf-gs7vs"}}
{"level":"error","ts":"2026-11-02T10:22:25.271631053Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-98gpc","name":"test-preemption-trainjob-pzmfq","reconcileID":"500d7751-fb29-4b88-93bb-a88b997bee81","job":"test-ns-98gpc/test-preemption-trainjob-pzmfq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-pzmfq\" is forbidden: unable to create new content in namespace test-ns-98gpc because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-02T10:22:25.271690999Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-98gpc","name":"test-preemption-trainjob-pzmfq","reconcileID":"500d7751-fb29-4b88-93bb-a88b997bee81","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-pzmfq\" is forbidden: unable to create new content in namespace test-ns-98gpc because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:25.347575863Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:141","msg":"ResourceFlavor create event","replica-role":"leader","resourceFlavor":{"name":"rf-qvx85"}}
{"level":"Level(-2)","ts":"2026-11-02T10:22:25.347710423Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-qvx85","reconcileID":"081cf190-518a-4a67-b248-a5a6249d3381"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:25.470654446Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:351","msg":"ClusterQueue create event","replica-role":"leader","clusterQueue":{"name":"cq-4scgv"}}
{"level":"Level(-2)","ts":"2026-11-02T10:22:25.470897777Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-4scgv","reconcileID":"4c931d24-88a7-4aba-aef0-2372626c2288"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:25.48849913Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:240","msg":"LocalQueue create event","replica-role":"leader","localQueue":{"name":"lq-t8596","namespace":"test-ns-dcr64"}}
{"level":"Level(-2)","ts":"2026-11-02T10:22:25.488605817Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-dcr64","name":"lq-t8596","reconcileID":"2fa1acf2-b10e-4e2c-81e6-0900f5c03c3b"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:25.52864713Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-4scgv"}}
{"level":"Level(-2)","ts":"2026-11-02T10:22:25.52880245Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-4scgv","reconcileID":"ef4f144e-0ffc-4248-bb16-b4a03e0b137a"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:25.542857236Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-t8596","namespace":"test-ns-dcr64"}}
{"level":"Level(-2)","ts":"2026-11-02T10:22:25.543082698Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-dcr64","name":"lq-t8596","reconcileID":"a2dbfc5b-987c-450d-bd13-be0a7e76a5aa"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:25.592222424Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-98gpc","name":"test-preemption-trainjob-pzmfq","reconcileID":"6763170c-a173-4fec-877d-669e5f76ad2b","job":"test-ns-98gpc/test-preemption-trainjob-pzmfq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:25.599607017Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-t8596","namespace":"test-ns-dcr64"}}
{"level":"Level(-2)","ts":"2026-11-02T10:22:25.599859923Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-dcr64","name":"lq-t8596","reconcileID":"133b7400-e680-4a33-8ca4-46dde9ba635d"}
{"level":"info","ts":"2026-11-02T10:22:25.60907967Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:117","msg":"Validating create","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-runtimepatches-trainjob-wnvxp","namespace":"test-ns-dcr64"},"namespace":"test-ns-dcr64","name":"test-runtimepatches-trainjob-wnvxp","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:admin","requestID":"82fb5774-f7e9-40ac-a2bc-bca998d8f28e","replica-role":"leader"}
{"level":"error","ts":"2026-11-02T10:22:25.643837641Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-98gpc","name":"test-preemption-trainjob-pzmfq","reconcileID":"6763170c-a173-4fec-877d-669e5f76ad2b","job":"test-ns-98gpc/test-preemption-trainjob-pzmfq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-pzmfq\" is forbidden: unable to create new content in namespace test-ns-98gpc because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-02T10:22:25.643903282Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-98gpc","name":"test-preemption-trainjob-pzmfq","reconcileID":"6763170c-a173-4fec-877d-669e5f76ad2b","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-pzmfq\" is forbidden: unable to create new content in namespace test-ns-98gpc because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:25.66255052Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-dcr64","name":"test-runtimepatches-trainjob-wnvxp","reconcileID":"0d2b20c7-ed81-40f9-bbda-ddb014a5f34c","job":"test-ns-dcr64/test-runtimepatches-trainjob-wnvxp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:25.977620299Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-dcr64","name":"trainjob-test-runtimepatches-trainjob-wnvxp-38f10","reconcileID":"ed2a3ffd-73d6-4c61-99bf-15da8e4227c9"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:25.977664345Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1276","msg":"Workload create event","replica-role":"leader","workload":{"name":"trainjob-test-runtimepatches-trainjob-wnvxp-38f10","namespace":"test-ns-dcr64"},"queue":"lq-t8596","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:25.977675353Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-dcr64","name":"trainjob-test-runtimepatches-trainjob-wnvxp-38f10","reconcileID":"ed2a3ffd-73d6-4c61-99bf-15da8e4227c9","isDeleted":false}
{"level":"debug","ts":"2026-11-02T10:22:25.977620915Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"test-runtimepatches-trainjob-wnvxp","namespace":"test-ns-dcr64"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"CreatedWorkload","action":"CreatedWorkload","note":"Created Workload: test-ns-dcr64/trainjob-test-runtimepatches-trainjob-wnvxp-38f10"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:25.977630082Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-dcr64","name":"test-runtimepatches-trainjob-wnvxp","reconcileID":"37849b7e-4d38-4cb8-9a0c-b9d61d48e0db","job":"test-ns-dcr64/test-runtimepatches-trainjob-wnvxp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:25.977832093Z","logger":"scheduler","caller":"scheduler/scheduler.go:308","msg":"Obtained heads","replica-role":"leader","schedulingCycle":4,"headCount":1,"waitDuration":5.89359144}
{"level":"Level(-2)","ts":"2026-11-02T10:22:25.977873503Z","logger":"scheduler","caller":"scheduler/scheduler.go:323","msg":"Snapshot taken","replica-role":"leader","schedulingCycle":4,"duration":0.000012523}
{"level":"Level(-2)","ts":"2026-11-02T10:22:25.977846927Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-dcr64","name":"trainjob-test-runtimepatches-trainjob-wnvxp-38f10","reconcileID":"9e0f2a61-c9be-4b03-8456-4fa84a72beb7"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:25.977967665Z","logger":"scheduler","caller":"scheduler/scheduler.go:328","msg":"Nomination done","replica-role":"leader","schedulingCycle":4,"entries":1,"inadmissibleEntries":0,"duration":0.000083761}
{"level":"Level(-2)","ts":"2026-11-02T10:22:25.97799116Z","logger":"scheduler","caller":"scheduler/scheduler.go:406","msg":"Attempting to schedule workload","replica-role":"leader","schedulingCycle":4,"workload":{"name":"trainjob-test-runtimepatches-trainjob-wnvxp-38f10","namespace":"test-ns-dcr64"},"clusterQueue":{"name":"cq-4scgv"}}
{"level":"Level(-2)","ts":"2026-11-02T10:22:25.978081771Z","logger":"scheduler","caller":"scheduler/scheduler.go:1041","msg":"Workload assumed in the cache","replica-role":"leader","schedulingCycle":4,"workload":{"name":"trainjob-test-runtimepatches-trainjob-wnvxp-38f10","namespace":"test-ns-dcr64"},"clusterQueue":{"name":"cq-4scgv"}}
{"level":"Level(-2)","ts":"2026-11-02T10:22:25.978137908Z","logger":"scheduler","caller":"scheduler/scheduler.go:362","msg":"Workload processing done","replica-role":"leader","schedulingCycle":4,"duration":0.000151456}
{"level":"Level(-2)","ts":"2026-11-02T10:22:25.978158654Z","logger":"scheduler","caller":"scheduler/scheduler.go:297","msg":"Scheduling cycle complete","replica-role":"leader","schedulingCycle":4,"duration":5.893918374}
{"level":"Level(-2)","ts":"2026-11-02T10:22:25.97817654Z","logger":"scheduler","caller":"scheduler/scheduler.go:295","msg":"Scheduling cycle starts","replica-role":"leader","schedulingCycle":5}
{"level":"Level(-2)","ts":"2026-11-02T10:22:26.088134128Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-dcr64","name":"trainjob-test-runtimepatches-trainjob-wnvxp-38f10","reconcileID":"3bc1da17-c696-4a08-a214-ce658cf72642"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:26.088149504Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-runtimepatches-trainjob-wnvxp-38f10","namespace":"test-ns-dcr64"},"queue":"lq-t8596","status":"admitted","prevStatus":"pending","clusterQueue":"cq-4scgv"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:26.08819884Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-dcr64","name":"trainjob-test-runtimepatches-trainjob-wnvxp-38f10","reconcileID":"3bc1da17-c696-4a08-a214-ce658cf72642","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T10:22:26.088222176Z","logger":"scheduler","caller":"scheduler/scheduler.go:1000","msg":"Workload successfully admitted and assigned flavors","replica-role":"leader","schedulingCycle":4,"workload":{"name":"trainjob-test-runtimepatches-trainjob-wnvxp-38f10","namespace":"test-ns-dcr64"},"clusterQueue":{"name":"cq-4scgv"},"assignments":[{"name":"node","flavors":{"cpu":"rf-qvx85","memory":"rf-qvx85"},"resourceUsage":{"cpu":"100m","memory":"128Mi"},"count":1}]}
{"level":"debug","ts":"2026-11-02T10:22:26.088232175Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-test-runtimepatches-trainjob-wnvxp-38f10","namespace":"test-ns-dcr64"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"QuotaReserved","action":"QuotaReserved","note":"Quota reserved in ClusterQueue cq-4scgv, wait time since queued was 1s"}
{"level":"debug","ts":"2026-11-02T10:22:26.088264914Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-test-runtimepatches-trainjob-wnvxp-38f10","namespace":"test-ns-dcr64"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"Admitted","action":"Admitted","note":"Admitted by ClusterQueue cq-4scgv, wait time since reservation was 1s"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:26.088353409Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-dcr64","name":"trainjob-test-runtimepatches-trainjob-wnvxp-38f10","reconcileID":"1c94ac7a-31a9-4b85-8490-bc47c533bf88"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:26.134712892Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-dcr64","name":"test-runtimepatches-trainjob-wnvxp","reconcileID":"44fe5432-3e31-498c-9675-242530c372ee","job":"test-ns-dcr64/test-runtimepatches-trainjob-wnvxp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:26.29221231Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-dcr64","name":"test-runtimepatches-trainjob-wnvxp","reconcileID":"44fe5432-3e31-498c-9675-242530c372ee","job":"test-ns-dcr64/test-runtimepatches-trainjob-wnvxp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-3)","ts":"2026-11-02T10:22:26.398774422Z","logger":"trainjob","caller":"jobframework/reconciler.go:594","msg":"Unsuspending job","replica-role":"leader","namespace":"test-ns-dcr64","name":"test-runtimepatches-trainjob-wnvxp","reconcileID":"44fe5432-3e31-498c-9675-242530c372ee","job":"test-ns-dcr64/test-runtimepatches-trainjob-wnvxp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-runtimepatches-trainjob-wnvxp\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-3)","ts":"2026-11-02T10:22:26.398855284Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-dcr64","name":"test-runtimepatches-trainjob-wnvxp","reconcileID":"44fe5432-3e31-498c-9675-242530c372ee","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-runtimepatches-trainjob-wnvxp\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:26.398958558Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-dcr64","name":"test-runtimepatches-trainjob-wnvxp","reconcileID":"f7b18c3c-9042-40d5-aacd-8546362b8691","job":"test-ns-dcr64/test-runtimepatches-trainjob-wnvxp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:26.55581629Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-dcr64","name":"test-runtimepatches-trainjob-wnvxp","reconcileID":"f7b18c3c-9042-40d5-aacd-8546362b8691","job":"test-ns-dcr64/test-runtimepatches-trainjob-wnvxp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"info","ts":"2026-11-02T10:22:26.664161348Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-runtimepatches-trainjob-wnvxp","namespace":"test-ns-dcr64"},"namespace":"test-ns-dcr64","name":"test-runtimepatches-trainjob-wnvxp","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"cfbe8ccb-295c-43ba-a5af-2569012cee5f","replica-role":"leader"}
{"level":"info","ts":"2026-11-02T10:22:26.827951296Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-runtimepatches-trainjob-wnvxp","namespace":"test-ns-dcr64"},"namespace":"test-ns-dcr64","name":"test-runtimepatches-trainjob-wnvxp","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"6290d036-aad7-4a89-ac61-4451b9450204","replica-role":"leader"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:26.830910541Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-4scgv"}}
{"level":"Level(-2)","ts":"2026-11-02T10:22:26.830987498Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-4scgv","reconcileID":"806d43b8-2efb-463b-be84-b96b0e7d71d7"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:26.831079585Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-dcr64","name":"trainjob-test-runtimepatches-trainjob-wnvxp-38f10","reconcileID":"563ef78e-eda1-443d-8d72-3149789a0be7"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:26.848507131Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:171","msg":"ResourceFlavor update event","replica-role":"leader","resourceFlavor":{"name":"rf-qvx85"}}
{"level":"Level(-2)","ts":"2026-11-02T10:22:26.848575888Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-qvx85","reconcileID":"05318532-bfb6-4fdb-9c9e-d4cee4eb6825"}
{"level":"debug","ts":"2026-11-02T10:22:26.884138765Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"test-runtimepatches-trainjob-wnvxp","namespace":"test-ns-dcr64"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"Started","action":"Started","note":"Admitted by clusterQueue cq-4scgv"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:26.884170834Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-98gpc","name":"test-preemption-trainjob-pzmfq","reconcileID":"a96353ec-77c9-45b7-8923-4c7719b3e3fa","job":"test-ns-98gpc/test-preemption-trainjob-pzmfq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:26.888029104Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-4scgv"}}
{"level":"Level(-2)","ts":"2026-11-02T10:22:26.888072511Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-dcr64","name":"lq-t8596","reconcileID":"a2b3b5b0-b549-4dca-83b4-376e4e11813c"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:26.888120635Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-4scgv","reconcileID":"60e80425-f146-46ed-a3e4-636cfe7b7e4d"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:26.888176332Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-dcr64","name":"trainjob-test-runtimepatches-trainjob-wnvxp-38f10","reconcileID":"ee77e19b-1d11-43b2-a8da-c8fbb0aa35c0"}
{"level":"error","ts":"2026-11-02T10:22:26.935161214Z","logger":"events","caller":"events/event_broadcaster.go:270","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-dcr64","name":"test-runtimepatches-trainjob-wnvxp.18e4334a8cdaaee7"},"error":"events.events.k8s.io \"test-runtimepatches-trainjob-wnvxp.18e4334a8cdaaee7\" is forbidden: unable to create new content in namespace test-ns-dcr64 because it is being terminated","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/workspace/upstream/kueue/src/vendor/k8s.io/client-go/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).attemptRecording\n\t/workspace/upstream/kueue/src/vendor/k8s.io/client-go/tools/events/event_broadcaster.go:211\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).recordToSink.func1\n\t/workspace/upstream/kueue/src/vendor/k8s.io/client-go/tools/events/event_broadcaster.go:200"}
{"level":"error","ts":"2026-11-02T10:22:26.935596915Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-98gpc","name":"test-preemption-trainjob-pzmfq","reconcileID":"a96353ec-77c9-45b7-8923-4c7719b3e3fa","job":"test-ns-98gpc/test-preemption-trainjob-pzmfq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-pzmfq\" is forbidden: unable to create new content in namespace test-ns-98gpc because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-02T10:22:26.935657529Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-98gpc","name":"test-preemption-trainjob-pzmfq","reconcileID":"a96353ec-77c9-45b7-8923-4c7719b3e3fa","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-pzmfq\" is forbidden: unable to create new content in namespace test-ns-98gpc because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:26.935762603Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-dcr64","name":"test-runtimepatches-trainjob-wnvxp","reconcileID":"238efdcd-f6c1-4029-906c-f37a812c51f8","job":"test-ns-dcr64/test-runtimepatches-trainjob-wnvxp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:26.94280461Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-t8596","namespace":"test-ns-dcr64"}}
{"level":"Level(-2)","ts":"2026-11-02T10:22:26.942891484Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-dcr64","name":"lq-t8596","reconcileID":"27b3c1b4-b5f0-4c48-98ff-7ccba34c05b0"}
{"level":"error","ts":"2026-11-02T10:22:26.987324227Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-dcr64","name":"test-runtimepatches-trainjob-wnvxp","reconcileID":"238efdcd-f6c1-4029-906c-f37a812c51f8","job":"test-ns-dcr64/test-runtimepatches-trainjob-wnvxp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-runtimepatches-trainjob-wnvxp\" is forbidden: unable to create new content in namespace test-ns-dcr64 because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-02T10:22:26.987388367Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-dcr64","name":"test-runtimepatches-trainjob-wnvxp","reconcileID":"238efdcd-f6c1-4029-906c-f37a812c51f8","error":"workloads.kueue.x-k8s.io \"test-runtimepatches-trainjob-wnvxp\" is forbidden: unable to create new content in namespace test-ns-dcr64 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:26.992657637Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-dcr64","name":"test-runtimepatches-trainjob-wnvxp","reconcileID":"cfa1bf6c-8eba-48fa-b0c7-6f0695194c91","job":"test-ns-dcr64/test-runtimepatches-trainjob-wnvxp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-11-02T10:22:27.044459182Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-dcr64","name":"test-runtimepatches-trainjob-wnvxp","reconcileID":"cfa1bf6c-8eba-48fa-b0c7-6f0695194c91","job":"test-ns-dcr64/test-runtimepatches-trainjob-wnvxp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-runtimepatches-trainjob-wnvxp\" is forbidden: unable to create new content in namespace test-ns-dcr64 because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-02T10:22:27.044517306Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-dcr64","name":"test-runtimepatches-trainjob-wnvxp","reconcileID":"cfa1bf6c-8eba-48fa-b0c7-6f0695194c91","error":"workloads.kueue.x-k8s.io \"test-runtimepatches-trainjob-wnvxp\" is forbidden: unable to create new content in namespace test-ns-dcr64 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:27.044598764Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-dcr64","name":"test-runtimepatches-trainjob-wnvxp","reconcileID":"6b71f7c8-8879-47ed-9b1e-de554eb58c64","job":"test-ns-dcr64/test-runtimepatches-trainjob-wnvxp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-11-02T10:22:27.096492927Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-dcr64","name":"test-runtimepatches-trainjob-wnvxp","reconcileID":"6b71f7c8-8879-47ed-9b1e-de554eb58c64","job":"test-ns-dcr64/test-runtimepatches-trainjob-wnvxp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-runtimepatches-trainjob-wnvxp\" is forbidden: unable to create new content in namespace test-ns-dcr64 because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-02T10:22:27.096550616Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-dcr64","name":"test-runtimepatches-trainjob-wnvxp","reconcileID":"6b71f7c8-8879-47ed-9b1e-de554eb58c64","error":"workloads.kueue.x-k8s.io \"test-runtimepatches-trainjob-wnvxp\" is forbidden: unable to create new content in namespace test-ns-dcr64 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T10:22:27.111843593Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-dcr64","name":"test-runtimepatches-trainjob-wnvxp","reconcileID":"8349dc2a-0ccb-465f-81a3-dacce5056f30","job":"test-ns-dcr64/test-runtimepatches-trainjob-wnvxp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}