{"level":"Level(-2)","ts":"2026-10-30T23:42:39.746272749Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-4h8x8","name":"e2e-deadline-job","reconcileID":"587c87ad-d19d-4ac7-acd0-9cddbf6bbb34","job":"e2e-4h8x8/e2e-deadline-job","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-4h8x8"}
{"level":"Level(-2)","ts":"2026-10-30T23:42:45.110889802Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-4h8x8","name":"e2e-deadline-job","reconcileID":"19e86a98-6cb1-439c-8ced-425e4259da7b","job":"e2e-4h8x8/e2e-deadline-job","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-4h8x8"}
{"level":"Level(-2)","ts":"2026-10-30T23:42:45.177471283Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-4h8x8","name":"e2e-deadline-job","reconcileID":"71b0df60-3963-4ccb-9c66-301b94fe6e1e","job":"e2e-4h8x8/e2e-deadline-job","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-4h8x8"}
{"level":"Level(-2)","ts":"2026-10-30T23:42:45.742407657Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-p4ppk","name":"e2e-test-runtime-update","reconcileID":"7a1a1e0b-ac5f-46da-8105-ba919a6233be","job":"e2e-p4ppk/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-p4ppk"}
{"level":"Level(-2)","ts":"2026-10-30T23:42:45.959743143Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-p4ppk","name":"e2e-test-runtime-update","reconcileID":"f52b6b42-c4bb-44c3-9370-c91ba5228b48","job":"e2e-p4ppk/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-p4ppk"}
{"level":"Level(-2)","ts":"2026-10-30T23:42:46.183059254Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-p4ppk","name":"e2e-test-runtime-update","reconcileID":"a7ccf8fe-4645-4467-8fd0-005933f8642b","job":"e2e-p4ppk/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-p4ppk"}
{"level":"Level(-2)","ts":"2026-10-30T23:42:46.260219291Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-p4ppk","name":"e2e-test-runtime-update","reconcileID":"9787bb9a-4666-49d6-8972-678329bd63d9","job":"e2e-p4ppk/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-p4ppk"}
{"level":"Level(-2)","ts":"2026-10-30T23:42:46.294707227Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-p4ppk","name":"e2e-test-runtime-update","reconcileID":"bcf89dd1-aecc-47c8-8896-27afaa7f1b9d","job":"e2e-p4ppk/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-p4ppk"}
{"level":"Level(-2)","ts":"2026-10-30T23:42:46.406817178Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-p4ppk","name":"e2e-test-runtime-update","reconcileID":"9e3cf987-bdfb-44cc-a62e-a122643556a9","job":"e2e-p4ppk/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-p4ppk"}
{"level":"Level(-2)","ts":"2026-10-30T23:42:46.695860072Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-p4ppk","name":"e2e-test-runtime-update","reconcileID":"52bda9e7-2380-46d0-ba0d-4a089847495a","job":"e2e-p4ppk/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-p4ppk"}
{"level":"Level(-2)","ts":"2026-10-30T23:42:46.805885932Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-p4ppk","name":"e2e-test-runtime-update","reconcileID":"bc4c2973-23ea-42d5-bc54-45e5e3eaaf69","job":"e2e-p4ppk/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-p4ppk"}
{"level":"Level(-2)","ts":"2026-10-30T23:42:46.915551969Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-p4ppk","name":"e2e-test-runtime-update","reconcileID":"739559fe-9128-4c68-bb64-d62da256d11a","job":"e2e-p4ppk/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-p4ppk"}
{"level":"Level(-2)","ts":"2026-10-30T23:42:46.924935472Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-p4ppk","name":"e2e-test-runtime-update","reconcileID":"ab7645e8-2598-46f1-b309-a12b6de7cc82","job":"e2e-p4ppk/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-p4ppk"}
{"level":"Level(-2)","ts":"2026-10-30T23:42:46.981409704Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-p4ppk","name":"e2e-test-runtime-update","reconcileID":"de0152a8-1a3f-40d3-b1f6-e2f4f407e851","job":"e2e-p4ppk/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-p4ppk"}
{"level":"Level(-2)","ts":"2026-10-30T23:42:47.191541237Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-p4ppk","name":"e2e-test-runtime-update","reconcileID":"45959b1c-d54a-4fca-b6ca-da6b74a6f500","job":"e2e-p4ppk/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-p4ppk"}
{"level":"Level(-2)","ts":"2026-10-30T23:42:47.472917398Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-p4ppk","name":"e2e-test-runtime-update","reconcileID":"0755f570-49e7-49e0-8ea5-266cfbc85afc","job":"e2e-p4ppk/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-p4ppk"}
{"level":"Level(-2)","ts":"2026-10-30T23:42:47.632330216Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-p4ppk","name":"e2e-test-runtime-update","reconcileID":"39d8d605-f8dd-4145-accd-3582c59af4a2","job":"e2e-p4ppk/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-p4ppk"}
{"level":"Level(-2)","ts":"2026-10-30T23:42:47.747767783Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-p4ppk","name":"e2e-test-runtime-update","reconcileID":"c35caea1-c12e-4414-9dd6-54b260b14b03","job":"e2e-p4ppk/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-p4ppk"}
{"level":"Level(-2)","ts":"2026-10-30T23:42:47.852607717Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-p4ppk","name":"e2e-test-runtime-update","reconcileID":"9d7a46b2-064e-4d7b-a242-b53b9fc02de1","job":"e2e-p4ppk/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-p4ppk"}
{"level":"Level(-2)","ts":"2026-10-30T23:42:47.910373367Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-p4ppk","name":"e2e-test-runtime-update","reconcileID":"3a43f26e-6ea4-4b27-9525-00c0d5d5647d","job":"e2e-p4ppk/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-p4ppk"}
{"level":"Level(-2)","ts":"2026-10-30T23:42:47.979669113Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-p4ppk","name":"e2e-test-runtime-update","reconcileID":"b3711cd5-a1a3-4141-b2f4-f96b7be937b4","job":"e2e-p4ppk/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-p4ppk"}
{"level":"Level(-2)","ts":"2026-10-30T23:42:48.02069788Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-p4ppk","name":"e2e-test-runtime-update","reconcileID":"dcd28006-77b4-40a4-a7e6-bccb84471967","job":"e2e-p4ppk/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-p4ppk"}
{"level":"Level(-2)","ts":"2026-10-30T23:42:48.131095588Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-p4ppk","name":"e2e-test-runtime-update","reconcileID":"82942e39-79fd-434a-ab1c-c7c216a8f1c1","job":"e2e-p4ppk/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-p4ppk"}
{"level":"Level(-2)","ts":"2026-10-30T23:42:48.716609649Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-bbx9g","name":"e2e-test-runtime-update","reconcileID":"1cbeb29c-920f-4010-bd30-1e11c7a3b6f5","job":"e2e-bbx9g/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-bbx9g"}
{"level":"Level(-2)","ts":"2026-10-30T23:42:48.938706801Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-bbx9g","name":"e2e-test-runtime-update","reconcileID":"d8ae7bd4-c9fb-4b59-bbb2-ce58780a2c45","job":"e2e-bbx9g/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-bbx9g"}
{"level":"Level(-2)","ts":"2026-10-30T23:42:49.111590682Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-bbx9g","name":"e2e-test-runtime-update","reconcileID":"494ded78-d75b-4922-a06f-1214dbf68e41","job":"e2e-bbx9g/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-bbx9g"}
{"level":"Level(-2)","ts":"2026-10-30T23:42:49.223587413Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-bbx9g","name":"e2e-test-runtime-update","reconcileID":"7660506d-f36d-4256-966c-6286c57e9576","job":"e2e-bbx9g/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-bbx9g"}
{"level":"Level(-2)","ts":"2026-10-30T23:42:49.23660321Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-bbx9g","name":"e2e-test-runtime-update","reconcileID":"a62b6078-4b49-45bb-9a82-074ff3d453b2","job":"e2e-bbx9g/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-bbx9g"}
{"level":"Level(-2)","ts":"2026-10-30T23:42:49.333677209Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-bbx9g","name":"e2e-test-runtime-update","reconcileID":"2a5f7f05-49cb-468a-8e9e-ee0f67b9c1a9","job":"e2e-bbx9g/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-bbx9g"}
{"level":"Level(-2)","ts":"2026-10-30T23:42:49.670882107Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-bbx9g","name":"e2e-test-runtime-update","reconcileID":"95e42798-86f7-4695-aaa9-9e508209610c","job":"e2e-bbx9g/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-bbx9g"}
{"level":"Level(-2)","ts":"2026-10-30T23:42:49.785434481Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-bbx9g","name":"e2e-test-runtime-update","reconcileID":"56796600-68a1-4834-a28e-b46ac78485d8","job":"e2e-bbx9g/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-bbx9g"}
{"level":"Level(-2)","ts":"2026-10-30T23:42:49.90101233Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-bbx9g","name":"e2e-test-runtime-update","reconcileID":"cde2dd43-073e-4d5d-a8f1-4c859d7a4fae","job":"e2e-bbx9g/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-bbx9g"}
{"level":"Level(-2)","ts":"2026-10-30T23:42:49.904465669Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-bbx9g","name":"e2e-test-runtime-update","reconcileID":"d3400599-e2db-4c69-bebb-8141c7135688","job":"e2e-bbx9g/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-bbx9g"}
{"level":"Level(-2)","ts":"2026-10-30T23:42:49.962695749Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-bbx9g","name":"e2e-test-runtime-update","reconcileID":"91b67988-4ee1-442c-baf8-8c40cd5d01ec","job":"e2e-bbx9g/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-bbx9g"}
{"level":"Level(-2)","ts":"2026-10-30T23:42:50.177547414Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-bbx9g","name":"e2e-test-runtime-update","reconcileID":"2e566233-4e7b-44cc-8737-627711b41438","job":"e2e-bbx9g/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-bbx9g"}
{"level":"Level(-2)","ts":"2026-10-30T23:42:50.447953993Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-bbx9g","name":"e2e-test-runtime-update","reconcileID":"6e9f304a-73bd-41b3-8f56-ea8cb8593d0b","job":"e2e-bbx9g/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-bbx9g"}
{"level":"Level(-2)","ts":"2026-10-30T23:42:50.617905417Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-bbx9g","name":"e2e-test-runtime-update","reconcileID":"49175476-bdbe-4aa8-9a73-5cd81aa8751a","job":"e2e-bbx9g/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-bbx9g"}
{"level":"Level(-2)","ts":"2026-10-30T23:42:50.727711304Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-bbx9g","name":"e2e-test-runtime-update","reconcileID":"4891d5cf-d519-4d05-b661-7fdda9383a4c","job":"e2e-bbx9g/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-bbx9g"}
{"level":"Level(-2)","ts":"2026-10-30T23:42:50.807678759Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-bbx9g","name":"e2e-test-runtime-update","reconcileID":"63acbdfa-bbba-400c-bc2e-96927544730c","job":"e2e-bbx9g/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-bbx9g"}
{"level":"Level(-2)","ts":"2026-10-30T23:42:50.867875736Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-bbx9g","name":"e2e-test-runtime-update","reconcileID":"0d882628-b843-410e-bdca-de85ad6b95dd","job":"e2e-bbx9g/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-bbx9g"}
{"level":"Level(-2)","ts":"2026-10-30T23:42:50.936236881Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-bbx9g","name":"e2e-test-runtime-update","reconcileID":"f20d9070-bcc9-439b-a8aa-55e84583db35","job":"e2e-bbx9g/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-bbx9g"}
{"level":"Level(-2)","ts":"2026-10-30T23:42:50.94503346Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-bbx9g","name":"e2e-test-runtime-update","reconcileID":"6f0cbbba-7f4c-4027-9a17-b2eb19fa3c42","job":"e2e-bbx9g/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-bbx9g"}
{"level":"Level(-2)","ts":"2026-10-30T23:42:51.057371377Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-bbx9g","name":"e2e-test-runtime-update","reconcileID":"a514fb9f-f612-4f48-9b3c-21e823ad6332","job":"e2e-bbx9g/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-bbx9g"}
{"level":"Level(-2)","ts":"2026-10-30T23:43:47.407125639Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-kdbxc","name":"test-dra-trainjob-d6slh","reconcileID":"f7cf349e-49f3-4ca7-82fa-6d976130f643","job":"test-ns-kdbxc/test-dra-trainjob-d6slh","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-kdbxc"}
{"level":"Level(-2)","ts":"2026-10-30T23:43:47.642189052Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-kdbxc","name":"test-dra-trainjob-d6slh","reconcileID":"4d5ebed5-f287-4b11-99e0-e506bba1c158","job":"test-ns-kdbxc/test-dra-trainjob-d6slh","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-kdbxc"}
{"level":"Level(-2)","ts":"2026-10-30T23:43:47.858651242Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-kdbxc","name":"test-dra-trainjob-d6slh","reconcileID":"34daed38-0772-4574-86b9-db53f0869d66","job":"test-ns-kdbxc/test-dra-trainjob-d6slh","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-kdbxc"}
{"level":"Level(-2)","ts":"2026-10-30T23:43:47.972846148Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-kdbxc","name":"test-dra-trainjob-d6slh","reconcileID":"089719b5-7ba5-4d69-9176-60bec2fcc53f","job":"test-ns-kdbxc/test-dra-trainjob-d6slh","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-kdbxc"}
{"level":"Level(-2)","ts":"2026-10-30T23:43:48.07071276Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-kdbxc","name":"test-dra-trainjob-d6slh","reconcileID":"00d85a52-49cf-445a-af30-04f242259b4f","job":"test-ns-kdbxc/test-dra-trainjob-d6slh","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-kdbxc"}
{"level":"Level(-2)","ts":"2026-10-30T23:43:48.184142965Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-kdbxc","name":"test-dra-trainjob-d6slh","reconcileID":"f2a3c9f6-5222-4081-ad73-2088327e3bd5","job":"test-ns-kdbxc/test-dra-trainjob-d6slh","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-kdbxc"}
{"level":"Level(-2)","ts":"2026-10-30T23:43:52.657133792Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ccfp6","name":"test-trainjob-k96pn","reconcileID":"e7cf1675-5a5e-4d47-b378-9087d226eccb","job":"test-ns-ccfp6/test-trainjob-k96pn","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ccfp6"}
{"level":"Level(-2)","ts":"2026-10-30T23:43:52.885582312Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ccfp6","name":"test-trainjob-k96pn","reconcileID":"f155e2c3-ce3e-4b87-bf9b-945f5fe8d15d","job":"test-ns-ccfp6/test-trainjob-k96pn","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ccfp6"}
{"level":"Level(-2)","ts":"2026-10-30T23:43:53.111370239Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ccfp6","name":"test-trainjob-k96pn","reconcileID":"174a2471-8331-4fbe-b31c-500b16d4d66d","job":"test-ns-ccfp6/test-trainjob-k96pn","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ccfp6"}
{"level":"Level(-2)","ts":"2026-10-30T23:43:53.224871736Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ccfp6","name":"test-trainjob-k96pn","reconcileID":"452cceeb-55b5-4634-a4f9-9d76cdd4c762","job":"test-ns-ccfp6/test-trainjob-k96pn","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ccfp6"}
{"level":"Level(-2)","ts":"2026-10-30T23:43:53.228990332Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ccfp6","name":"test-trainjob-k96pn","reconcileID":"02e97eaa-476b-438f-850c-6fbb4b6dec7c","job":"test-ns-ccfp6/test-trainjob-k96pn","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ccfp6"}
{"level":"Level(-2)","ts":"2026-10-30T23:43:53.3410378Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ccfp6","name":"test-trainjob-k96pn","reconcileID":"8aa65749-c4c5-4020-9776-6731ee77d142","job":"test-ns-ccfp6/test-trainjob-k96pn","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ccfp6"}
{"level":"Level(-2)","ts":"2026-10-30T23:43:54.703182506Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-kdbxc","name":"test-dra-trainjob-d6slh","reconcileID":"530b90c5-f6c1-44de-bedb-1184f6440183","job":"test-ns-kdbxc/test-dra-trainjob-d6slh","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-kdbxc"}
{"level":"Level(-2)","ts":"2026-10-30T23:43:54.815411232Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-kdbxc","name":"test-dra-trainjob-d6slh","reconcileID":"4be5d20d-b9ea-4271-bd8d-75462d06ea94","job":"test-ns-kdbxc/test-dra-trainjob-d6slh","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-kdbxc"}
{"level":"Level(-2)","ts":"2026-10-30T23:43:56.041615912Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ccfp6","name":"test-trainjob-k96pn","reconcileID":"c13f01d7-0e85-4833-b7ab-f1e9bd8deaaa","job":"test-ns-ccfp6/test-trainjob-k96pn","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ccfp6"}
{"level":"Level(-2)","ts":"2026-10-30T23:43:56.153240159Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ccfp6","name":"test-trainjob-k96pn","reconcileID":"1e7744f6-8e81-4605-871a-c55c43323927","job":"test-ns-ccfp6/test-trainjob-k96pn","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ccfp6"}
{"level":"Level(-2)","ts":"2026-10-30T23:43:58.789621015Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-kdbxc","name":"test-dra-trainjob-d6slh","reconcileID":"26a8ffa7-b313-46c0-8ce3-66b81df681bc","job":"test-ns-kdbxc/test-dra-trainjob-d6slh","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-kdbxc"}
{"level":"Level(-2)","ts":"2026-10-30T23:44:04.096339718Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ccfp6","name":"test-trainjob-k96pn","reconcileID":"fda32e90-8920-4790-9ee4-d1d83f0ff3e1","job":"test-ns-ccfp6/test-trainjob-k96pn","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ccfp6"}
{"level":"Level(-2)","ts":"2026-10-30T23:44:04.209817658Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ccfp6","name":"test-trainjob-k96pn","reconcileID":"93544a87-cd85-4c6a-8b98-0b461c4c4cef","job":"test-ns-ccfp6/test-trainjob-k96pn","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ccfp6"}
{"level":"Level(-2)","ts":"2026-10-30T23:44:05.101929415Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ccfp6","name":"test-trainjob-k96pn","reconcileID":"c4d27132-9b51-4da6-a712-4889fc10ad20","job":"test-ns-ccfp6/test-trainjob-k96pn","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ccfp6"}
{"level":"Level(-2)","ts":"2026-10-30T23:44:05.215160758Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ccfp6","name":"test-trainjob-k96pn","reconcileID":"e8d45deb-f8e4-4b44-996f-79e2ec0064d5","job":"test-ns-ccfp6/test-trainjob-k96pn","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ccfp6"}
{"level":"Level(-2)","ts":"2026-10-30T23:44:05.220431823Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ccfp6","name":"test-trainjob-k96pn","reconcileID":"de0c964d-f5ef-427d-a08f-a79160b9710a","job":"test-ns-ccfp6/test-trainjob-k96pn","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ccfp6"}
{"level":"Level(-2)","ts":"2026-10-30T23:44:05.334015067Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ccfp6","name":"test-trainjob-k96pn","reconcileID":"327e3f5c-8b3c-4faf-bb1c-04268168f783","job":"test-ns-ccfp6/test-trainjob-k96pn","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ccfp6"}
{"level":"Level(-2)","ts":"2026-10-30T23:44:06.092841363Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-gjd8n","name":"test-trainjob-l28bw","reconcileID":"299d3ca9-83b6-49ae-a657-b0d1ed72641c","job":"test-ns-gjd8n/test-trainjob-l28bw","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-gjd8n"}
{"level":"Level(-2)","ts":"2026-10-30T23:44:06.325565326Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-gjd8n","name":"test-trainjob-l28bw","reconcileID":"dff1afe0-2d57-4573-93e0-5cb1adc5440e","job":"test-ns-gjd8n/test-trainjob-l28bw","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-gjd8n"}
{"level":"Level(-2)","ts":"2026-10-30T23:44:06.553036286Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-gjd8n","name":"test-trainjob-l28bw","reconcileID":"a183d7bf-b3b5-43c0-9f02-03f895ebf9cd","job":"test-ns-gjd8n/test-trainjob-l28bw","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-gjd8n"}
{"level":"Level(-2)","ts":"2026-10-30T23:44:06.636399438Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-gjd8n","name":"test-trainjob-l28bw","reconcileID":"2a5dcda9-c87d-4acf-b64b-0fdfb455ab5d","job":"test-ns-gjd8n/test-trainjob-l28bw","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-gjd8n"}
{"level":"Level(-2)","ts":"2026-10-30T23:44:06.669103412Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-gjd8n","name":"test-trainjob-l28bw","reconcileID":"92cbebdc-c1cc-4a4d-9b0d-59ba50470c3a","job":"test-ns-gjd8n/test-trainjob-l28bw","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-gjd8n"}
{"level":"Level(-2)","ts":"2026-10-30T23:44:06.785007509Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-gjd8n","name":"test-trainjob-l28bw","reconcileID":"794a6234-724a-4fb7-9087-e57c4adde98b","job":"test-ns-gjd8n/test-trainjob-l28bw","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-gjd8n"}
{"level":"Level(-2)","ts":"2026-10-30T23:59:19.721108632Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-gjd8n","name":"test-trainjob-l28bw","reconcileID":"0b9ac696-93ff-465b-a246-ae5a0436dc7a","job":"test-ns-gjd8n/test-trainjob-l28bw","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-gjd8n"}
{"level":"Level(-2)","ts":"2026-10-30T23:59:19.833645493Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-gjd8n","name":"test-trainjob-l28bw","reconcileID":"4a8b881a-6480-4bfd-b1c9-47e0add3c91b","job":"test-ns-gjd8n/test-trainjob-l28bw","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-gjd8n"}
{"level":"Level(-2)","ts":"2026-10-30T23:59:27.705759672Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-gjd8n","name":"test-trainjob-l28bw","reconcileID":"8883e729-c6dd-4c08-be33-9dd7ab1105b9","job":"test-ns-gjd8n/test-trainjob-l28bw","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-gjd8n"}
{"level":"Level(-2)","ts":"2026-10-30T23:59:27.818197951Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-gjd8n","name":"test-trainjob-l28bw","reconcileID":"f6d5a24b-ca12-4589-a4f7-c2fdb84e1c03","job":"test-ns-gjd8n/test-trainjob-l28bw","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-gjd8n"}
{"level":"Level(-2)","ts":"2026-10-30T23:59:28.715837954Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-gjd8n","name":"test-trainjob-l28bw","reconcileID":"82e76695-17ca-4262-8e84-0dcdced7591b","job":"test-ns-gjd8n/test-trainjob-l28bw","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-gjd8n"}
{"level":"Level(-2)","ts":"2026-10-30T23:59:28.83058404Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-gjd8n","name":"test-trainjob-l28bw","reconcileID":"50c145f7-b059-41d9-8c1e-471cc7750ea0","job":"test-ns-gjd8n/test-trainjob-l28bw","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-gjd8n"}
{"level":"Level(-2)","ts":"2026-10-30T23:59:28.835539599Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-gjd8n","name":"test-trainjob-l28bw","reconcileID":"f5985acd-6d6e-481b-a29a-a59797a814a0","job":"test-ns-gjd8n/test-trainjob-l28bw","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-gjd8n"}
{"level":"Level(-2)","ts":"2026-10-30T23:59:28.941775Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-gjd8n","name":"test-trainjob-l28bw","reconcileID":"4e880692-f310-407f-96a8-91bfb504e826","job":"test-ns-gjd8n/test-trainjob-l28bw","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-gjd8n"}
{"level":"Level(-2)","ts":"2026-10-30T23:59:30.132357398Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-zjwl7","name":"test-trainjob-g5ft4","reconcileID":"49a649cc-216d-44cf-8318-ca68ce69704c","job":"test-ns-zjwl7/test-trainjob-g5ft4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-zjwl7"}
{"level":"Level(-2)","ts":"2026-10-30T23:59:30.578678474Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-zjwl7","name":"test-trainjob-g5ft4","reconcileID":"16570c07-471a-4ef0-a05b-3acbcf121d6a","job":"test-ns-zjwl7/test-trainjob-g5ft4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-zjwl7"}
{"level":"Level(-2)","ts":"2026-10-30T23:59:30.798806759Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-zjwl7","name":"test-trainjob-g5ft4","reconcileID":"11bc4859-e5e7-4bc4-a90d-71ca8c30019b","job":"test-ns-zjwl7/test-trainjob-g5ft4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-zjwl7"}
{"level":"Level(-2)","ts":"2026-10-30T23:59:30.911924763Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-zjwl7","name":"test-trainjob-g5ft4","reconcileID":"df743139-e8e2-4da1-867b-b37ff42ed4b6","job":"test-ns-zjwl7/test-trainjob-g5ft4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-zjwl7"}
{"level":"Level(-2)","ts":"2026-10-30T23:59:31.105754846Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-zjwl7","name":"test-trainjob-g5ft4","reconcileID":"87a9e461-9af0-4739-83ac-8e22d49f67fe","job":"test-ns-zjwl7/test-trainjob-g5ft4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-zjwl7"}
{"level":"Level(-2)","ts":"2026-10-30T23:59:31.216313106Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-zjwl7","name":"test-trainjob-g5ft4","reconcileID":"b445647c-44c6-4ec9-a418-2b7a17134dd0","job":"test-ns-zjwl7/test-trainjob-g5ft4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-zjwl7"}
{"level":"Level(-2)","ts":"2026-10-31T00:02:14.634468666Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-zjwl7","name":"test-trainjob-g5ft4","reconcileID":"d7195be1-b68f-45ef-85be-af63f61de3ef","job":"test-ns-zjwl7/test-trainjob-g5ft4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-zjwl7"}
{"level":"Level(-2)","ts":"2026-10-31T00:02:14.747858806Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-zjwl7","name":"test-trainjob-g5ft4","reconcileID":"00bc26f8-a683-462e-8241-e983f2c1d736","job":"test-ns-zjwl7/test-trainjob-g5ft4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-zjwl7"}
{"level":"Level(-2)","ts":"2026-10-31T00:02:17.639484073Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-zjwl7","name":"test-trainjob-g5ft4","reconcileID":"3cf6327f-c8e5-4e56-9b3c-a4c617b0e48c","job":"test-ns-zjwl7/test-trainjob-g5ft4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-zjwl7"}
{"level":"Level(-2)","ts":"2026-10-31T00:02:17.753208125Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-zjwl7","name":"test-trainjob-g5ft4","reconcileID":"cfa1e35f-ed5a-4679-bb57-dbea90fa5187","job":"test-ns-zjwl7/test-trainjob-g5ft4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-zjwl7"}
{"level":"Level(-2)","ts":"2026-10-31T00:02:18.651075821Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-zjwl7","name":"test-trainjob-g5ft4","reconcileID":"72313c46-52ba-4665-a5bb-ac19b0984c36","job":"test-ns-zjwl7/test-trainjob-g5ft4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-zjwl7"}
{"level":"Level(-2)","ts":"2026-10-31T00:02:18.762650801Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-zjwl7","name":"test-trainjob-g5ft4","reconcileID":"d32f7cf3-7d83-464d-b537-90c26e64f72c","job":"test-ns-zjwl7/test-trainjob-g5ft4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-zjwl7"}
{"level":"Level(-2)","ts":"2026-10-31T00:02:18.769388152Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-zjwl7","name":"test-trainjob-g5ft4","reconcileID":"964214aa-59aa-400e-a6e8-39efe57dd363","job":"test-ns-zjwl7/test-trainjob-g5ft4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-zjwl7"}
{"level":"Level(-2)","ts":"2026-10-31T00:02:18.87718935Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-zjwl7","name":"test-trainjob-g5ft4","reconcileID":"d4537a04-6b93-4018-a942-02656dbd61b8","job":"test-ns-zjwl7/test-trainjob-g5ft4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-zjwl7"}
{"level":"Level(-2)","ts":"2026-10-31T00:02:20.040788712Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-pg6k9","name":"test-trainjob-8crdp","reconcileID":"d4993f6b-5074-42e3-aef8-73336b3d42d1","job":"test-ns-pg6k9/test-trainjob-8crdp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-pg6k9"}
{"level":"Level(-2)","ts":"2026-10-31T00:02:20.625437856Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-pg6k9","name":"test-trainjob-8crdp","reconcileID":"1d2fc0b1-133e-412b-9867-82891b8e0b74","job":"test-ns-pg6k9/test-trainjob-8crdp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-pg6k9"}
{"level":"Level(-2)","ts":"2026-10-31T00:02:20.803681144Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-pg6k9","name":"test-trainjob-8crdp","reconcileID":"b9b7629c-4362-498d-8e64-d2030c0cb7b1","job":"test-ns-pg6k9/test-trainjob-8crdp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-pg6k9"}
{"level":"Level(-2)","ts":"2026-10-31T00:02:20.917905799Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-pg6k9","name":"test-trainjob-8crdp","reconcileID":"e953a30f-757a-4d7f-8aff-6f562596108c","job":"test-ns-pg6k9/test-trainjob-8crdp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-pg6k9"}
{"level":"Level(-2)","ts":"2026-10-31T00:02:21.154349636Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-pg6k9","name":"test-trainjob-8crdp","reconcileID":"ac81618e-03fc-44b7-965c-2ff9050abbf8","job":"test-ns-pg6k9/test-trainjob-8crdp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-pg6k9"}
{"level":"Level(-2)","ts":"2026-10-31T00:02:21.27010244Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-pg6k9","name":"test-trainjob-8crdp","reconcileID":"ae7bab1f-bf75-4f16-b96e-f104b04c17fd","job":"test-ns-pg6k9/test-trainjob-8crdp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-pg6k9"}
{"level":"Level(-2)","ts":"2026-10-31T00:02:24.464344941Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-zjwl7","name":"test-trainjob-g5ft4","reconcileID":"f4422c3a-ba16-445d-a806-0ab60a96e326","job":"test-ns-zjwl7/test-trainjob-g5ft4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-zjwl7"}
{"level":"Level(-2)","ts":"2026-10-31T00:02:24.576035614Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-zjwl7","name":"test-trainjob-g5ft4","reconcileID":"f3239ca2-2341-4254-9358-7891d8d66a8a","job":"test-ns-zjwl7/test-trainjob-g5ft4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-zjwl7"}
{"level":"Level(-2)","ts":"2026-10-31T00:02:24.621457622Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-gjd8n","name":"test-trainjob-l28bw","reconcileID":"dff356c9-e717-421c-a1de-4491c32918b1","job":"test-ns-gjd8n/test-trainjob-l28bw","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-gjd8n"}
{"level":"Level(-2)","ts":"2026-10-31T00:02:25.008207853Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-ccfp6","name":"test-trainjob-k96pn","reconcileID":"3b42f512-4483-42d3-b68a-24f8b2a19b9c","job":"test-ns-ccfp6/test-trainjob-k96pn","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-ccfp6"}
{"level":"Level(-2)","ts":"2026-10-31T00:02:43.786167787Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-pg6k9","name":"test-trainjob-8crdp","reconcileID":"60eddf99-7475-429e-b849-96eca1f3e83e","job":"test-ns-pg6k9/test-trainjob-8crdp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-pg6k9"}
{"level":"Level(-2)","ts":"2026-10-31T00:02:43.902250528Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-pg6k9","name":"test-trainjob-8crdp","reconcileID":"c56e3b28-4f5b-41f6-8db0-309e8fe44b40","job":"test-ns-pg6k9/test-trainjob-8crdp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-pg6k9"}
{"level":"Level(-2)","ts":"2026-10-31T00:02:48.810391723Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-pg6k9","name":"test-trainjob-8crdp","reconcileID":"86cddac6-7ce2-4e6d-bf14-b595d9c35ad9","job":"test-ns-pg6k9/test-trainjob-8crdp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-pg6k9"}
{"level":"Level(-2)","ts":"2026-10-31T00:02:48.926678135Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-pg6k9","name":"test-trainjob-8crdp","reconcileID":"7cc7af20-7935-46ce-a975-d677ab17442e","job":"test-ns-pg6k9/test-trainjob-8crdp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-pg6k9"}
{"level":"Level(-2)","ts":"2026-10-31T00:02:49.820537914Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-pg6k9","name":"test-trainjob-8crdp","reconcileID":"b0fe34a0-b78d-41e0-8b42-f97e1ee5995b","job":"test-ns-pg6k9/test-trainjob-8crdp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-pg6k9"}
{"level":"Level(-2)","ts":"2026-10-31T00:02:49.938709654Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-pg6k9","name":"test-trainjob-8crdp","reconcileID":"0a08b76e-176d-41bd-905f-84b9b0eba715","job":"test-ns-pg6k9/test-trainjob-8crdp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-pg6k9"}
{"level":"Level(-2)","ts":"2026-10-31T00:02:49.983887641Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-pg6k9","name":"test-trainjob-8crdp","reconcileID":"f3a03de0-5067-4fa5-a96e-092007729e42","job":"test-ns-pg6k9/test-trainjob-8crdp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-pg6k9"}
{"level":"Level(-2)","ts":"2026-10-31T00:02:50.056184977Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-pg6k9","name":"test-trainjob-8crdp","reconcileID":"d83a0863-0815-4e81-88da-a014a6c3f04b","job":"test-ns-pg6k9/test-trainjob-8crdp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-pg6k9"}
{"level":"Level(-2)","ts":"2026-10-31T00:02:50.125841511Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-pg6k9","name":"test-trainjob-8crdp","reconcileID":"ada363fc-db08-42b6-b562-242fa570772c","job":"test-ns-pg6k9/test-trainjob-8crdp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-pg6k9"}
{"level":"Level(-2)","ts":"2026-10-31T00:02:50.28574881Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-pg6k9","name":"test-trainjob-8crdp","reconcileID":"900cd156-ea94-4c5a-83ca-9973138d1dc6","job":"test-ns-pg6k9/test-trainjob-8crdp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-pg6k9"}
{"level":"Level(-2)","ts":"2026-10-31T00:05:51.741572925Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-pg6k9","name":"test-trainjob-8crdp","reconcileID":"344b80d4-26ff-4d4a-8d86-250f6ebef0a5","job":"test-ns-pg6k9/test-trainjob-8crdp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-pg6k9"}
{"level":"Level(-2)","ts":"2026-10-31T00:05:51.859721556Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-pg6k9","name":"test-trainjob-8crdp","reconcileID":"75b316a0-d97c-4450-a119-729c7eab9499","job":"test-ns-pg6k9/test-trainjob-8crdp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-pg6k9"}
{"level":"Level(-2)","ts":"2026-10-31T00:06:02.829072478Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-pg6k9","name":"test-trainjob-8crdp","reconcileID":"97117dec-5b49-413a-b441-e8037ffc05b6","job":"test-ns-pg6k9/test-trainjob-8crdp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-pg6k9"}
{"level":"Level(-2)","ts":"2026-10-31T00:06:02.943862332Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-pg6k9","name":"test-trainjob-8crdp","reconcileID":"4cccffb5-c2e4-4f19-a2e0-9768df61efcc","job":"test-ns-pg6k9/test-trainjob-8crdp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-pg6k9"}
{"level":"Level(-2)","ts":"2026-10-31T00:06:03.79840585Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-pg6k9","name":"test-trainjob-8crdp","reconcileID":"fb434498-d65b-4a0a-834b-9c6d0f4b7bb7","job":"test-ns-pg6k9/test-trainjob-8crdp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-pg6k9"}
{"level":"Level(-2)","ts":"2026-10-31T00:06:03.914176404Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-pg6k9","name":"test-trainjob-8crdp","reconcileID":"5bc68c3e-8b71-4a61-88e4-d65a50c35f4b","job":"test-ns-pg6k9/test-trainjob-8crdp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-pg6k9"}
{"level":"Level(-2)","ts":"2026-10-31T00:06:04.022027474Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-pg6k9","name":"test-trainjob-8crdp","reconcileID":"fcc27894-c9e6-4b2b-998b-a5b85b2bfa77","job":"test-ns-pg6k9/test-trainjob-8crdp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-pg6k9"}
{"level":"Level(-2)","ts":"2026-10-31T00:06:04.136575696Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-pg6k9","name":"test-trainjob-8crdp","reconcileID":"80fbd663-689c-45d0-ba7c-66af63cdf54c","job":"test-ns-pg6k9/test-trainjob-8crdp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-pg6k9"}
{"level":"Level(-2)","ts":"2026-10-31T00:06:04.323545668Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-pg6k9","name":"test-trainjob-8crdp","reconcileID":"ef3edc75-01f2-4b2a-8d57-03141500beaf","job":"test-ns-pg6k9/test-trainjob-8crdp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-pg6k9"}
{"level":"Level(-2)","ts":"2026-10-31T00:06:04.44171883Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-pg6k9","name":"test-trainjob-8crdp","reconcileID":"7fa3cc96-11cc-48c4-961f-f56a6add0e2d","job":"test-ns-pg6k9/test-trainjob-8crdp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-pg6k9"}
{"level":"Level(-2)","ts":"2026-10-31T00:06:10.994101923Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-pg6k9","name":"test-trainjob-8crdp","reconcileID":"3de9c693-a00d-4c42-a23a-d9edf4adaf9b","job":"test-ns-pg6k9/test-trainjob-8crdp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-pg6k9"}
{"level":"Level(-2)","ts":"2026-10-31T00:06:11.11089722Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-pg6k9","name":"test-trainjob-8crdp","reconcileID":"24612bb9-ce46-4201-adf6-401b1b97dcc4","job":"test-ns-pg6k9/test-trainjob-8crdp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-pg6k9"}
{"level":"Level(-2)","ts":"2026-10-31T00:06:11.153690019Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-pg6k9","name":"test-trainjob-8crdp","reconcileID":"b0a6e102-4025-4571-94a8-9c0014b2e439","job":"test-ns-pg6k9/test-trainjob-8crdp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-pg6k9"}
{"level":"Level(-2)","ts":"2026-10-31T00:06:11.226393808Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-pg6k9","name":"test-trainjob-8crdp","reconcileID":"da8f66bd-db41-4955-9cee-f42fcb32b51f","job":"test-ns-pg6k9/test-trainjob-8crdp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-pg6k9"}
{"level":"Level(-2)","ts":"2026-10-31T00:06:17.087921022Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-pg6k9","name":"test-trainjob-8crdp","reconcileID":"bd5eb834-3f05-49e5-a372-d737db3e85ca","job":"test-ns-pg6k9/test-trainjob-8crdp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-pg6k9"}
{"level":"Level(-2)","ts":"2026-10-31T00:06:17.204185862Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-pg6k9","name":"test-trainjob-8crdp","reconcileID":"71ba82f6-4e2d-472a-9505-08ee83f1162a","job":"test-ns-pg6k9/test-trainjob-8crdp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-pg6k9"}
{"level":"Level(-2)","ts":"2026-10-31T00:06:28.338370226Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-s4mbk","name":"j9298137b8c4","reconcileID":"44c7bfdc-6b0e-4340-af95-58458bcc1b46","job":"test-ns-s4mbk/j9298137b8c4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-s4mbk"}
{"level":"Level(-2)","ts":"2026-10-31T00:06:28.788679976Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-s4mbk","name":"j9298137b8c4","reconcileID":"a662f94e-0785-4711-ba22-38767639442a","job":"test-ns-s4mbk/j9298137b8c4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-s4mbk"}
{"level":"Level(-2)","ts":"2026-10-31T00:06:29.012430524Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-s4mbk","name":"j9298137b8c4","reconcileID":"2e01df09-a8d8-494c-bbe3-af311b3db6cd","job":"test-ns-s4mbk/j9298137b8c4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-s4mbk"}
{"level":"Level(-2)","ts":"2026-10-31T00:06:29.1265863Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-s4mbk","name":"j9298137b8c4","reconcileID":"3f4aa6e6-eb39-4340-80e9-c3ef628b77c2","job":"test-ns-s4mbk/j9298137b8c4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-s4mbk"}
{"level":"Level(-2)","ts":"2026-10-31T00:06:29.212306968Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-s4mbk","name":"j9298137b8c4","reconcileID":"1661c51d-5606-4b3e-9f1b-a8985e31d7d9","job":"test-ns-s4mbk/j9298137b8c4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-s4mbk"}
{"level":"Level(-2)","ts":"2026-10-31T00:06:29.327057294Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-s4mbk","name":"j9298137b8c4","reconcileID":"5c74633c-6f0e-48a0-a14d-e457c04e9149","job":"test-ns-s4mbk/j9298137b8c4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-s4mbk"}
{"level":"Level(-2)","ts":"2026-10-31T00:06:31.19149192Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-s4mbk","name":"j9298137b8c4","reconcileID":"0090133b-63a3-4c1c-b53e-4c676b016ff6","job":"test-ns-s4mbk/j9298137b8c4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-s4mbk"}
{"level":"Level(-2)","ts":"2026-10-31T00:06:31.306838791Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-s4mbk","name":"j9298137b8c4","reconcileID":"c41c372e-44da-46ef-ad13-c13c1206d003","job":"test-ns-s4mbk/j9298137b8c4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-s4mbk"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:04.226340205Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-s4mbk","name":"j9298137b8c4","reconcileID":"eb5b5aa1-2f9f-4978-bf28-a40a51d11e44","job":"test-ns-s4mbk/j9298137b8c4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-s4mbk"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:04.369380936Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-s4mbk","name":"j9298137b8c4","reconcileID":"89a8608c-a88f-420d-ab01-d5f1be4ce979","job":"test-ns-s4mbk/j9298137b8c4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-s4mbk"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:05.235495488Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-s4mbk","name":"j9298137b8c4","reconcileID":"aefd0146-2ea9-4af7-b4d6-fe552fd66dd1","job":"test-ns-s4mbk/j9298137b8c4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-s4mbk"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:05.348558461Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-s4mbk","name":"j9298137b8c4","reconcileID":"b19e5669-63ba-44f6-a58f-67fa3fdf95f2","job":"test-ns-s4mbk/j9298137b8c4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-s4mbk"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:05.390128573Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-s4mbk","name":"j9298137b8c4","reconcileID":"062f466c-08bd-4b28-9f70-bd3f619e98c4","job":"test-ns-s4mbk/j9298137b8c4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-s4mbk"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:05.4628615Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-s4mbk","name":"j9298137b8c4","reconcileID":"1f73c24c-047c-46f4-93b7-7b37ea68a78d","job":"test-ns-s4mbk/j9298137b8c4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-s4mbk"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:10.907833167Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:141","msg":"ResourceFlavor create event","replica-role":"leader","resourceFlavor":{"name":"rf-dltsh"}}
{"level":"Level(-2)","ts":"2026-10-31T00:07:10.90798969Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-dltsh","reconcileID":"301e9237-5d30-4b95-b1c6-365d5c92d80e"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:11.25418792Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:351","msg":"ClusterQueue create event","replica-role":"leader","clusterQueue":{"name":"cq-ttv4t"}}
{"level":"Level(-2)","ts":"2026-10-31T00:07:11.254458902Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-ttv4t","reconcileID":"d9bc827d-3898-45fe-86ce-4585018b112e"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:11.273053242Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:240","msg":"LocalQueue create event","replica-role":"leader","localQueue":{"name":"lq-2gslv","namespace":"test-ns-cjgqr"}}
{"level":"Level(-2)","ts":"2026-10-31T00:07:11.273205555Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-cjgqr","name":"lq-2gslv","reconcileID":"cc7fc652-2989-494a-929b-daacbb23c5f7"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:11.311685341Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-ttv4t"}}
{"level":"Level(-2)","ts":"2026-10-31T00:07:11.311958982Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-ttv4t","reconcileID":"532d91ee-6b65-4d4c-9954-89bd83ebe0d8"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:11.331347485Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-cjgqr","name":"lq-2gslv","reconcileID":"390d7a03-03e8-4ef5-a2c2-ac603361f927"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:11.33134765Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-2gslv","namespace":"test-ns-cjgqr"}}
{"level":"Level(-2)","ts":"2026-10-31T00:07:11.388476262Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-cjgqr","name":"lq-2gslv","reconcileID":"806a8ba0-7cf5-4c17-8e03-cc09fab909b4"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:11.388565672Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-2gslv","namespace":"test-ns-cjgqr"}}
{"level":"Level(-3)","ts":"2026-10-31T00:07:11.442058071Z","logger":"localqueue-reconciler","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-cjgqr","name":"lq-2gslv","reconcileID":"806a8ba0-7cf5-4c17-8e03-cc09fab909b4","error":"Operation cannot be fulfilled on localqueues.kueue.x-k8s.io \"lq-2gslv\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:11.442144059Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-cjgqr","name":"lq-2gslv","reconcileID":"8b84c9c0-3936-41ed-b6ef-59f5ceeb550f"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:12.114544103Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-cjgqr","name":"dw-deployment-2sxqb-698d7dbcbf-9vkbl","reconcileID":"d51286c9-782d-4503-be61-1ad2ee6328a9","job":"test-ns-cjgqr/dw-deployment-2sxqb-698d7dbcbf-9vkbl","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:12.168308931Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1.PriorityClass","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"debug","ts":"2026-10-31T00:07:12.600625239Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"dw-deployment-2sxqb-698d7dbcbf-9vkbl","namespace":"test-ns-cjgqr"},"kind":"Pod","apiVersion":"v1","type":"Normal","reason":"CreatedWorkload","action":"CreatedWorkload","note":"Created Workload: test-ns-cjgqr/pod-dw-deployment-2sxqb-698d7dbcbf-9vkbl-bf644"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:12.651581771Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1276","msg":"Workload create event","replica-role":"leader","workload":{"name":"pod-dw-deployment-2sxqb-698d7dbcbf-9vkbl-bf644","namespace":"test-ns-cjgqr"},"queue":"lq-2gslv","status":"pending"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:12.651638607Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-cjgqr","name":"dw-deployment-2sxqb-698d7dbcbf-9vkbl","reconcileID":"b77f23ad-4fa2-445c-9c86-b140f2419183","job":"test-ns-cjgqr/dw-deployment-2sxqb-698d7dbcbf-9vkbl","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:12.651591039Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-cjgqr","name":"pod-dw-deployment-2sxqb-698d7dbcbf-9vkbl-bf644","reconcileID":"744eaa30-5992-4e0f-ba8a-8bb3168c5359"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:12.651737358Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-cjgqr","name":"pod-dw-deployment-2sxqb-698d7dbcbf-9vkbl-bf644","reconcileID":"744eaa30-5992-4e0f-ba8a-8bb3168c5359","isDeleted":false}
{"level":"Level(-2)","ts":"2026-10-31T00:07:12.651838152Z","logger":"scheduler","caller":"scheduler/scheduler.go:308","msg":"Obtained heads","replica-role":"follower","schedulingCycle":1,"headCount":1,"waitDuration":2549.549540535}
{"level":"Level(-2)","ts":"2026-10-31T00:07:12.651872771Z","logger":"scheduler","caller":"scheduler/scheduler.go:323","msg":"Snapshot taken","replica-role":"follower","schedulingCycle":1,"duration":0.000010819}
{"level":"Level(-2)","ts":"2026-10-31T00:07:12.651942924Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-cjgqr","name":"pod-dw-deployment-2sxqb-698d7dbcbf-9vkbl-bf644","reconcileID":"0a437adb-3426-4040-a4ca-4521cfa047f5"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:12.651988648Z","logger":"scheduler","caller":"scheduler/scheduler.go:328","msg":"Nomination done","replica-role":"follower","schedulingCycle":1,"entries":1,"inadmissibleEntries":0,"duration":0.000101557}
{"level":"Level(-2)","ts":"2026-10-31T00:07:12.65201575Z","logger":"scheduler","caller":"scheduler/scheduler.go:406","msg":"Attempting to schedule workload","replica-role":"follower","schedulingCycle":1,"workload":{"name":"pod-dw-deployment-2sxqb-698d7dbcbf-9vkbl-bf644","namespace":"test-ns-cjgqr"},"clusterQueue":{"name":"cq-ttv4t"}}
{"level":"Level(-2)","ts":"2026-10-31T00:07:12.652108119Z","logger":"scheduler","caller":"scheduler/scheduler.go:1041","msg":"Workload assumed in the cache","replica-role":"follower","schedulingCycle":1,"workload":{"name":"pod-dw-deployment-2sxqb-698d7dbcbf-9vkbl-bf644","namespace":"test-ns-cjgqr"},"clusterQueue":{"name":"cq-ttv4t"}}
{"level":"Level(-2)","ts":"2026-10-31T00:07:12.652142568Z","logger":"scheduler","caller":"scheduler/scheduler.go:362","msg":"Workload processing done","replica-role":"follower","schedulingCycle":1,"duration":0.000132608}
{"level":"Level(-2)","ts":"2026-10-31T00:07:12.652156896Z","logger":"scheduler","caller":"scheduler/scheduler.go:297","msg":"Scheduling cycle complete","replica-role":"follower","schedulingCycle":1,"duration":2549.549859855}
{"level":"Level(-2)","ts":"2026-10-31T00:07:12.652171647Z","logger":"scheduler","caller":"scheduler/scheduler.go:295","msg":"Scheduling cycle starts","replica-role":"leader","schedulingCycle":2}
{"level":"Level(-2)","ts":"2026-10-31T00:07:12.768638015Z","logger":"scheduler","caller":"scheduler/scheduler.go:1000","msg":"Workload successfully admitted and assigned flavors","replica-role":"follower","schedulingCycle":1,"workload":{"name":"pod-dw-deployment-2sxqb-698d7dbcbf-9vkbl-bf644","namespace":"test-ns-cjgqr"},"clusterQueue":{"name":"cq-ttv4t"},"assignments":[{"name":"main","flavors":{"cpu":"rf-dltsh","memory":"rf-dltsh"},"resourceUsage":{"cpu":"1","memory":"3Gi"},"count":1}]}
{"level":"debug","ts":"2026-10-31T00:07:12.768694443Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"pod-dw-deployment-2sxqb-698d7dbcbf-9vkbl-bf644","namespace":"test-ns-cjgqr"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"QuotaReserved","action":"QuotaReserved","note":"Quota reserved in ClusterQueue cq-ttv4t, wait time since queued was 1s"}
{"level":"debug","ts":"2026-10-31T00:07:12.768728945Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"pod-dw-deployment-2sxqb-698d7dbcbf-9vkbl-bf644","namespace":"test-ns-cjgqr"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"Admitted","action":"Admitted","note":"Admitted by ClusterQueue cq-ttv4t, wait time since reservation was 1s"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:12.768800555Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"pod-dw-deployment-2sxqb-698d7dbcbf-9vkbl-bf644","namespace":"test-ns-cjgqr"},"queue":"lq-2gslv","status":"admitted","prevStatus":"pending","clusterQueue":"cq-ttv4t"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:12.76887502Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-cjgqr","name":"pod-dw-deployment-2sxqb-698d7dbcbf-9vkbl-bf644","reconcileID":"1b5f4d0c-cbbb-460a-8804-4d40f03f70cb"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:12.7689063Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-cjgqr","name":"dw-deployment-2sxqb-698d7dbcbf-9vkbl","reconcileID":"00b47285-54eb-4aa5-9256-5ee5f3fd374a","job":"test-ns-cjgqr/dw-deployment-2sxqb-698d7dbcbf-9vkbl","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:12.768935248Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-cjgqr","name":"pod-dw-deployment-2sxqb-698d7dbcbf-9vkbl-bf644","reconcileID":"1b5f4d0c-cbbb-460a-8804-4d40f03f70cb","isDeleted":false}
{"level":"Level(-2)","ts":"2026-10-31T00:07:12.769076722Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-cjgqr","name":"pod-dw-deployment-2sxqb-698d7dbcbf-9vkbl-bf644","reconcileID":"be064714-3cc4-4174-a247-0def7e8f4a33"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:12.769155008Z","logger":"v1_pod","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-cjgqr","name":"dw-deployment-2sxqb-698d7dbcbf-9vkbl","reconcileID":"00b47285-54eb-4aa5-9256-5ee5f3fd374a","job":"test-ns-cjgqr/dw-deployment-2sxqb-698d7dbcbf-9vkbl","gvk":"/v1, Kind=Pod"}
{"level":"debug","ts":"2026-10-31T00:07:12.88541134Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"dw-deployment-2sxqb-698d7dbcbf-9vkbl","namespace":"test-ns-cjgqr"},"kind":"Pod","apiVersion":"v1","type":"Normal","reason":"Started","action":"Started","note":"Admitted by clusterQueue cq-ttv4t"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:12.886085103Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-cjgqr","name":"dw-deployment-2sxqb-698d7dbcbf-9vkbl","reconcileID":"5b863d16-e8b9-4a33-be65-c4b706d89d74","job":"test-ns-cjgqr/dw-deployment-2sxqb-698d7dbcbf-9vkbl","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:12.892645329Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-cjgqr","name":"dw-deployment-2sxqb-698d7dbcbf-9vkbl","reconcileID":"356dc629-e9fa-4672-8e17-f1f4e5c8754a","job":"test-ns-cjgqr/dw-deployment-2sxqb-698d7dbcbf-9vkbl","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:12.952531708Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-cjgqr","name":"dw-deployment-2sxqb-698d7dbcbf-9vkbl","reconcileID":"83b06529-3003-40b5-898a-dfd19752bbcd","job":"test-ns-cjgqr/dw-deployment-2sxqb-698d7dbcbf-9vkbl","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:13.065556195Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-cjgqr","name":"dw-deployment-2sxqb-698d7dbcbf-9vkbl","reconcileID":"f88c31a1-3ea9-411d-b0f1-f2fd3ce7e1a0","job":"test-ns-cjgqr/dw-deployment-2sxqb-698d7dbcbf-9vkbl","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:13.652561803Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-ttv4t","reconcileID":"052b1f22-2534-44f7-a6d7-d8af23170f24"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:13.652561139Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-cjgqr","name":"lq-2gslv","reconcileID":"6f36dd6b-b879-41bf-939c-a391f99d5fa0"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:13.708606918Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-2gslv","namespace":"test-ns-cjgqr"}}
{"level":"Level(-2)","ts":"2026-10-31T00:07:13.708708973Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-cjgqr","name":"lq-2gslv","reconcileID":"e598d4c0-1a49-4be5-8f8c-8cbf7044f9eb"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:13.755075385Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-ttv4t"}}
{"level":"Level(-2)","ts":"2026-10-31T00:07:13.755240063Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-ttv4t","reconcileID":"60fa83dc-4f56-4a26-9564-f7bfa778a592"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:13.887083468Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-cjgqr","name":"pod-dw-deployment-2sxqb-698d7dbcbf-9vkbl-bf644","reconcileID":"6e0fdcbc-0925-4ed3-b427-d9295b69a84f"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:13.887865319Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-cjgqr","name":"dw-deployment-2sxqb-698d7dbcbf-9vkbl","reconcileID":"24ae76ae-d39d-47ed-b3ad-4bdf8f36f5f7","job":"test-ns-cjgqr/dw-deployment-2sxqb-698d7dbcbf-9vkbl","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:14.888637145Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-cjgqr","name":"pod-dw-deployment-2sxqb-698d7dbcbf-9vkbl-bf644","reconcileID":"0d656cbd-fd25-4792-a15b-2148b80e967a"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:15.17572369Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-cjgqr","name":"dw-deployment-2sxqb-698d7dbcbf-9vkbl","reconcileID":"6d84a416-f513-46c8-ad69-3d26d5db7139","job":"test-ns-cjgqr/dw-deployment-2sxqb-698d7dbcbf-9vkbl","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:16.176164271Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-cjgqr","name":"pod-dw-deployment-2sxqb-698d7dbcbf-9vkbl-bf644","reconcileID":"f3c234b6-a0a1-4c26-bec1-b6378c22b1e5"}
{"level":"info","ts":"2026-10-31T00:07:27.1516281Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:117","msg":"Validating create","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"d60dcb77863b","namespace":"test-ns-cjgqr"},"namespace":"test-ns-cjgqr","name":"d60dcb77863b","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:opendatahub:trainer-e2e","requestID":"9a3ecda4-4fb0-4878-9dac-47e06563e62f","replica-role":"leader"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:27.213009215Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-cjgqr","name":"d60dcb77863b","reconcileID":"1a7ad759-185c-4b45-8862-378f6d7e39a6","job":"test-ns-cjgqr/d60dcb77863b","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"debug","ts":"2026-10-31T00:07:27.53807316Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"d60dcb77863b","namespace":"test-ns-cjgqr"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"CreatedWorkload","action":"CreatedWorkload","note":"Created Workload: test-ns-cjgqr/trainjob-d60dcb77863b-e902a"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:27.538091475Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-cjgqr","name":"d60dcb77863b","reconcileID":"686afd4c-cf68-40c7-8933-5e00edf9a7fb","job":"test-ns-cjgqr/d60dcb77863b","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:27.538611491Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1276","msg":"Workload create event","replica-role":"leader","workload":{"name":"trainjob-d60dcb77863b-e902a","namespace":"test-ns-cjgqr"},"queue":"lq-2gslv","status":"pending"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:27.538715403Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-cjgqr","name":"trainjob-d60dcb77863b-e902a","reconcileID":"5c485f4d-f07b-4e6e-86ce-757de61e7ea4"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:27.538781099Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-cjgqr","name":"trainjob-d60dcb77863b-e902a","reconcileID":"5c485f4d-f07b-4e6e-86ce-757de61e7ea4","isDeleted":false}
{"level":"Level(-2)","ts":"2026-10-31T00:07:27.538833077Z","logger":"scheduler","caller":"scheduler/scheduler.go:308","msg":"Obtained heads","replica-role":"leader","schedulingCycle":2,"headCount":1,"waitDuration":14.886660676}
{"level":"Level(-2)","ts":"2026-10-31T00:07:27.53887633Z","logger":"scheduler","caller":"scheduler/scheduler.go:323","msg":"Snapshot taken","replica-role":"leader","schedulingCycle":2,"duration":0.000012535}
{"level":"Level(-2)","ts":"2026-10-31T00:07:27.538893074Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-cjgqr","name":"trainjob-d60dcb77863b-e902a","reconcileID":"e623f756-b286-4dce-aef2-8d107fbafec0"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:27.538963547Z","logger":"scheduler","caller":"scheduler/scheduler.go:328","msg":"Nomination done","replica-role":"leader","schedulingCycle":2,"entries":1,"inadmissibleEntries":0,"duration":0.000072031}
{"level":"Level(-2)","ts":"2026-10-31T00:07:27.539028Z","logger":"scheduler","caller":"scheduler/scheduler.go:406","msg":"Attempting to schedule workload","replica-role":"leader","schedulingCycle":2,"workload":{"name":"trainjob-d60dcb77863b-e902a","namespace":"test-ns-cjgqr"},"clusterQueue":{"name":"cq-ttv4t"}}
{"level":"Level(-2)","ts":"2026-10-31T00:07:27.539123187Z","logger":"scheduler","caller":"scheduler/scheduler.go:1041","msg":"Workload assumed in the cache","replica-role":"leader","schedulingCycle":2,"workload":{"name":"trainjob-d60dcb77863b-e902a","namespace":"test-ns-cjgqr"},"clusterQueue":{"name":"cq-ttv4t"}}
{"level":"Level(-2)","ts":"2026-10-31T00:07:27.539155714Z","logger":"scheduler","caller":"scheduler/scheduler.go:362","msg":"Workload processing done","replica-role":"leader","schedulingCycle":2,"duration":0.000136533}
{"level":"Level(-2)","ts":"2026-10-31T00:07:27.539177465Z","logger":"scheduler","caller":"scheduler/scheduler.go:297","msg":"Scheduling cycle complete","replica-role":"leader","schedulingCycle":2,"duration":14.887005532}
{"level":"Level(-2)","ts":"2026-10-31T00:07:27.539194543Z","logger":"scheduler","caller":"scheduler/scheduler.go:295","msg":"Scheduling cycle starts","replica-role":"leader","schedulingCycle":3}
{"level":"Level(-2)","ts":"2026-10-31T00:07:27.652015185Z","logger":"scheduler","caller":"scheduler/scheduler.go:1000","msg":"Workload successfully admitted and assigned flavors","replica-role":"leader","schedulingCycle":2,"workload":{"name":"trainjob-d60dcb77863b-e902a","namespace":"test-ns-cjgqr"},"clusterQueue":{"name":"cq-ttv4t"},"assignments":[{"name":"node","flavors":{"cpu":"rf-dltsh","memory":"rf-dltsh"},"resourceUsage":{"cpu":"4","memory":"16Gi"},"count":2}]}
{"level":"debug","ts":"2026-10-31T00:07:27.65203822Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-d60dcb77863b-e902a","namespace":"test-ns-cjgqr"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"QuotaReserved","action":"QuotaReserved","note":"Quota reserved in ClusterQueue cq-ttv4t, wait time since queued was 1s"}
{"level":"debug","ts":"2026-10-31T00:07:27.652336849Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-d60dcb77863b-e902a","namespace":"test-ns-cjgqr"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"Admitted","action":"Admitted","note":"Admitted by ClusterQueue cq-ttv4t, wait time since reservation was 1s"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:27.65290963Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-d60dcb77863b-e902a","namespace":"test-ns-cjgqr"},"queue":"lq-2gslv","status":"admitted","prevStatus":"pending","clusterQueue":"cq-ttv4t"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:27.653374879Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-cjgqr","name":"trainjob-d60dcb77863b-e902a","reconcileID":"a77934b6-7adf-4854-9f9a-c6e586080db1"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:27.65392833Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-cjgqr","name":"trainjob-d60dcb77863b-e902a","reconcileID":"62c9a86f-fe88-482c-aa50-055e287f050d"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:27.654044992Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-cjgqr","name":"trainjob-d60dcb77863b-e902a","reconcileID":"62c9a86f-fe88-482c-aa50-055e287f050d","isDeleted":false}
{"level":"error","ts":"2026-10-31T00:07:27.86266433Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-cjgqr","name":"d60dcb77863b","reconcileID":"686afd4c-cf68-40c7-8933-5e00edf9a7fb","error":"workloads.kueue.x-k8s.io \"trainjob-d60dcb77863b-e902a\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:27.862803115Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-cjgqr","name":"d60dcb77863b","reconcileID":"2fdc9b84-dcc9-4cf6-9463-0bc624607123","job":"test-ns-cjgqr/d60dcb77863b","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:28.022804858Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-cjgqr","name":"d60dcb77863b","reconcileID":"2fdc9b84-dcc9-4cf6-9463-0bc624607123","job":"test-ns-cjgqr/d60dcb77863b","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"info","ts":"2026-10-31T00:07:28.142015696Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"d60dcb77863b","namespace":"test-ns-cjgqr"},"namespace":"test-ns-cjgqr","name":"d60dcb77863b","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"d8b58e98-a263-41dc-aa4f-cab87e2d8bde","replica-role":"leader"}
{"level":"info","ts":"2026-10-31T00:07:28.31227736Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"d60dcb77863b","namespace":"test-ns-cjgqr"},"namespace":"test-ns-cjgqr","name":"d60dcb77863b","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"db932c20-e8dc-47b8-84a1-570640f0f95a","replica-role":"leader"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:28.370020503Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-cjgqr","name":"d60dcb77863b","reconcileID":"a1238c05-1be5-4689-ad48-11ceb7ba2182","job":"test-ns-cjgqr/d60dcb77863b","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"debug","ts":"2026-10-31T00:07:28.370018043Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"d60dcb77863b","namespace":"test-ns-cjgqr"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"Started","action":"Started","note":"Admitted by clusterQueue cq-ttv4t"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:28.530841437Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-cjgqr","name":"d60dcb77863b","reconcileID":"a1238c05-1be5-4689-ad48-11ceb7ba2182","job":"test-ns-cjgqr/d60dcb77863b","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:28.539132841Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-cjgqr","name":"lq-2gslv","reconcileID":"dc8ab43d-d017-4a0a-ba3f-801646c7638e"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:28.54022089Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-ttv4t","reconcileID":"e977aec4-673e-4584-8532-f0c7972b6fa5"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:28.596930315Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-2gslv","namespace":"test-ns-cjgqr"}}
{"level":"Level(-2)","ts":"2026-10-31T00:07:28.597068786Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-cjgqr","name":"lq-2gslv","reconcileID":"632bf1bb-f9a7-45f0-91e4-0592644f0d25"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:28.597756895Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-ttv4t"}}
{"level":"Level(-2)","ts":"2026-10-31T00:07:28.597895504Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-ttv4t","reconcileID":"d36ac1da-6a76-477b-8eb9-9534661c2117"}
{"level":"Level(-3)","ts":"2026-10-31T00:07:28.641822924Z","logger":"trainjob","caller":"jobframework/reconciler.go:594","msg":"Unsuspending job","replica-role":"leader","namespace":"test-ns-cjgqr","name":"d60dcb77863b","reconcileID":"a1238c05-1be5-4689-ad48-11ceb7ba2182","job":"test-ns-cjgqr/d60dcb77863b","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"d60dcb77863b\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-3)","ts":"2026-10-31T00:07:28.641888625Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-cjgqr","name":"d60dcb77863b","reconcileID":"a1238c05-1be5-4689-ad48-11ceb7ba2182","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"d60dcb77863b\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:28.641977071Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-cjgqr","name":"d60dcb77863b","reconcileID":"f1a11e3b-824e-4ed4-882b-26de4ef84023","job":"test-ns-cjgqr/d60dcb77863b","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:28.802252251Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-cjgqr","name":"d60dcb77863b","reconcileID":"b8a81066-0a09-424d-b518-d8af0bd43945","job":"test-ns-cjgqr/d60dcb77863b","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:28.962830311Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-cjgqr","name":"d60dcb77863b","reconcileID":"88cf9a1a-e35f-4c1e-9189-2d24f8da2889","job":"test-ns-cjgqr/d60dcb77863b","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:29.123014658Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-cjgqr","name":"d60dcb77863b","reconcileID":"4de4cd19-e2f6-49fe-b231-8466894edb39","job":"test-ns-cjgqr/d60dcb77863b","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:29.283354137Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-cjgqr","name":"d60dcb77863b","reconcileID":"6fb4b9c5-4775-47d0-8ad3-a24138697f4f","job":"test-ns-cjgqr/d60dcb77863b","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:29.443734708Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-cjgqr","name":"d60dcb77863b","reconcileID":"3e64ce42-64cb-4b0e-a5c8-005737c6f75d","job":"test-ns-cjgqr/d60dcb77863b","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:30.018880607Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-cjgqr","name":"trainjob-d60dcb77863b-e902a","reconcileID":"37aee788-4826-4b7d-85d6-7056e40efc94"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:31.122224753Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-cjgqr","name":"trainjob-d60dcb77863b-e902a","reconcileID":"22e838c9-f92d-4e2e-ba51-8de12cda28f7"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:32.369870981Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-cjgqr","name":"trainjob-d60dcb77863b-e902a","reconcileID":"5d7d5eba-26b0-4c2d-872d-0ee6f56a4bae"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:32.389582378Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-cjgqr","name":"d60dcb77863b","reconcileID":"a30c2d63-2b3d-47ea-92d1-4a32564dfd4d","job":"test-ns-cjgqr/d60dcb77863b","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T00:07:32.550374787Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-cjgqr","name":"d60dcb77863b","reconcileID":"690dd86e-a944-41f7-b48c-f57a56ab729c","job":"test-ns-cjgqr/d60dcb77863b","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:04.617118081Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-cjgqr","name":"trainjob-d60dcb77863b-e902a","reconcileID":"5f120a4c-39e1-4127-8ebf-7969845c1189"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:04.639389651Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-cjgqr","name":"d60dcb77863b","reconcileID":"20d5a10f-87d5-45bf-a345-e6781a5cf723","job":"test-ns-cjgqr/d60dcb77863b","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:04.800697861Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-cjgqr","name":"d60dcb77863b","reconcileID":"7d959998-2978-4a33-89c2-078905197e72","job":"test-ns-cjgqr/d60dcb77863b","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:05.647139328Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-cjgqr","name":"d60dcb77863b","reconcileID":"43530b9c-9334-42f8-8584-93a967f46f7d","job":"test-ns-cjgqr/d60dcb77863b","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:05.70823472Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-cjgqr","name":"trainjob-d60dcb77863b-e902a","reconcileID":"6d519a6c-2598-40cc-8e62-dcfe2480cd11"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:05.8085068Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-cjgqr","name":"d60dcb77863b","reconcileID":"a11d8af7-6d74-4705-a139-86de5c9673f0","job":"test-ns-cjgqr/d60dcb77863b","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:05.969765538Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-cjgqr","name":"d60dcb77863b","reconcileID":"451901ed-7946-479e-8d43-25301baa2c8c","job":"test-ns-cjgqr/d60dcb77863b","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"debug","ts":"2026-10-31T00:08:06.243216127Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"d60dcb77863b","namespace":"test-ns-cjgqr"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"FinishedWorkload","action":"FinishedWorkload","note":"Workload 'test-ns-cjgqr/trainjob-d60dcb77863b-e902a' is declared finished"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:06.24354081Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-cjgqr","name":"trainjob-d60dcb77863b-e902a","reconcileID":"cb69aee6-cce2-40a8-bb8a-3c0c23fc5d7e"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:06.243599539Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-cjgqr","name":"trainjob-d60dcb77863b-e902a","reconcileID":"cb69aee6-cce2-40a8-bb8a-3c0c23fc5d7e","isDeleted":false}
{"level":"Level(-2)","ts":"2026-10-31T00:08:06.243548997Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-d60dcb77863b-e902a","namespace":"test-ns-cjgqr"},"queue":"lq-2gslv","status":"finished","prevStatus":"admitted","clusterQueue":"cq-ttv4t"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:06.243684491Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-cjgqr","name":"d60dcb77863b","reconcileID":"8ba1a576-7f3f-4712-a4b4-8ffca8d78742","job":"test-ns-cjgqr/d60dcb77863b","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:06.243757945Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-cjgqr","name":"trainjob-d60dcb77863b-e902a","reconcileID":"02ce6755-67bd-4a4d-9b8c-0dcaf0b9a9dc"}
{"level":"debug","ts":"2026-10-31T00:08:06.403571863Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"d60dcb77863b","namespace":"test-ns-cjgqr"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"FinishedWorkload","action":"FinishedWorkload","note":"Workload 'test-ns-cjgqr/trainjob-d60dcb77863b-e902a' is declared finished"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:06.517486823Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-d60dcb77863b-e902a","namespace":"test-ns-cjgqr"},"queue":"lq-2gslv","status":"finished","clusterQueue":"cq-ttv4t"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:06.517546466Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-cjgqr","name":"trainjob-d60dcb77863b-e902a","reconcileID":"21ad4f81-d014-4fab-b050-a37161e99278"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:06.517610202Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-cjgqr","name":"trainjob-d60dcb77863b-e902a","reconcileID":"21ad4f81-d014-4fab-b050-a37161e99278","isDeleted":false}
{"level":"Level(-2)","ts":"2026-10-31T00:08:06.517604897Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-cjgqr","name":"d60dcb77863b","reconcileID":"52774564-14d2-4245-9a72-4ac1ea22ae66","job":"test-ns-cjgqr/d60dcb77863b","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:06.517655793Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-cjgqr","name":"trainjob-d60dcb77863b-e902a","reconcileID":"0b96865f-8718-415d-9a11-0ce5c5453ceb"}
{"level":"debug","ts":"2026-10-31T00:08:06.678134906Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"d60dcb77863b","namespace":"test-ns-cjgqr"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"FinishedWorkload","action":"FinishedWorkload","note":"Workload 'test-ns-cjgqr/trainjob-d60dcb77863b-e902a' is declared finished"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:07.244197795Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-ttv4t","reconcileID":"af3e848e-74c6-4464-8fcb-fd1d35e02fb6"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:07.244196546Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-cjgqr","name":"lq-2gslv","reconcileID":"710da777-7ca6-4627-8058-8a34d795f47a"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:07.30112227Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-2gslv","namespace":"test-ns-cjgqr"}}
{"level":"Level(-2)","ts":"2026-10-31T00:08:07.301239204Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-cjgqr","name":"lq-2gslv","reconcileID":"8cadee9f-28d3-49e0-bc9e-72aad3a281a6"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:07.30430333Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-ttv4t"}}
{"level":"Level(-2)","ts":"2026-10-31T00:08:07.304439409Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-ttv4t","reconcileID":"8e1856bf-2662-4239-996f-20addb23fd61"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:07.765320604Z","logger":"trainjob","caller":"workload/workload.go:1810","msg":"Workload is orphaned; finishing to release quota","replica-role":"leader","namespace":"test-ns-cjgqr","name":"d60dcb77863b","reconcileID":"e7ee6555-62fc-4917-86cf-7d1fe254f6a4","job":"test-ns-cjgqr/d60dcb77863b","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:07.81590853Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1312","msg":"Workload delete event","replica-role":"leader","workload":{"name":"trainjob-d60dcb77863b-e902a","namespace":"test-ns-cjgqr"},"queue":"lq-2gslv","status":"finished"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:07.815981609Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-cjgqr","name":"trainjob-d60dcb77863b-e902a","reconcileID":"24719801-7c30-4fd9-b9db-084478fdd4a0"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:07.816057546Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-cjgqr","name":"trainjob-d60dcb77863b-e902a","reconcileID":"24719801-7c30-4fd9-b9db-084478fdd4a0","isDeleted":true}
{"level":"Level(-2)","ts":"2026-10-31T00:08:08.81664208Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-cjgqr","name":"lq-2gslv","reconcileID":"6b16236f-e3ce-4e07-9e7f-db3c7f65aa05"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:08.816645373Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-ttv4t","reconcileID":"c2fafba3-80f9-47a2-9155-1521ca035eeb"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:08.839972913Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:171","msg":"ResourceFlavor update event","replica-role":"leader","resourceFlavor":{"name":"rf-dltsh"}}
{"level":"Level(-2)","ts":"2026-10-31T00:08:08.840074244Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-dltsh","reconcileID":"b8e7ac5e-7fda-4b75-af4d-86bca58f9722"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:08.840172434Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-ttv4t"}}
{"level":"Level(-2)","ts":"2026-10-31T00:08:08.840235445Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-ttv4t","reconcileID":"f041164c-3038-4b74-a6f1-e766f3605d75"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:08.840420231Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-cjgqr","name":"pod-dw-deployment-2sxqb-698d7dbcbf-9vkbl-bf644","reconcileID":"8437ae4a-5e11-4985-8c95-12e35cd4cbe0"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:08.898603271Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-ttv4t"}}
{"level":"Level(-2)","ts":"2026-10-31T00:08:08.898676294Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-ttv4t","reconcileID":"8504ac19-f1da-4bd6-b355-fba5f8cadc7a"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:08.898673583Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-cjgqr","name":"lq-2gslv","reconcileID":"4ee6554a-bc11-4224-b6d6-b599c49bbf61"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:08.898728451Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-cjgqr","name":"pod-dw-deployment-2sxqb-698d7dbcbf-9vkbl-bf644","reconcileID":"fab6ac59-7bdf-42fc-80a4-26a914997389"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:08.956827886Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-2gslv","namespace":"test-ns-cjgqr"}}
{"level":"Level(-2)","ts":"2026-10-31T00:08:08.957004866Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-cjgqr","name":"lq-2gslv","reconcileID":"99239f93-6fd9-4833-83bc-e600e6b677b0"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:09.210137767Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:141","msg":"ResourceFlavor create event","replica-role":"leader","resourceFlavor":{"name":"rf-2gxjk"}}
{"level":"Level(-2)","ts":"2026-10-31T00:08:09.210238781Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-2gxjk","reconcileID":"f5a95934-2753-4d3c-81f6-693c1c63cc5f"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:09.334625822Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:351","msg":"ClusterQueue create event","replica-role":"leader","clusterQueue":{"name":"cq-vfd72"}}
{"level":"Level(-2)","ts":"2026-10-31T00:08:09.334853219Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-vfd72","reconcileID":"0d1ae166-4f75-4cf2-b21a-fda21a6a01d4"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:09.353136818Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:240","msg":"LocalQueue create event","replica-role":"leader","localQueue":{"name":"lq-d7bs9","namespace":"test-ns-9p4tw"}}
{"level":"Level(-2)","ts":"2026-10-31T00:08:09.353258331Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-9p4tw","name":"lq-d7bs9","reconcileID":"0b63b9c4-b407-4b32-acc6-5f8153c35546"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:09.392792037Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-vfd72"}}
{"level":"Level(-2)","ts":"2026-10-31T00:08:09.392972805Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-vfd72","reconcileID":"677d124c-a73b-4f18-83e1-54785d4792af"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:09.409302357Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-d7bs9","namespace":"test-ns-9p4tw"}}
{"level":"Level(-2)","ts":"2026-10-31T00:08:09.409464378Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-9p4tw","name":"lq-d7bs9","reconcileID":"6b840408-b389-4a8e-8732-501d00245d91"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:09.467664163Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-d7bs9","namespace":"test-ns-9p4tw"}}
{"level":"Level(-2)","ts":"2026-10-31T00:08:09.467774868Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-9p4tw","name":"lq-d7bs9","reconcileID":"8fd7226f-1339-471e-9cd6-84d6d00858df"}
{"level":"info","ts":"2026-10-31T00:08:09.473175097Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:117","msg":"Validating create","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-preemption-trainjob-g9fzc","namespace":"test-ns-9p4tw"},"namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:admin","requestID":"b924f1e6-f431-4c99-b5ac-f0142168cd92","replica-role":"leader"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:09.527709301Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","reconcileID":"90344993-3f23-4e8a-bd64-3133114073db","job":"test-ns-9p4tw/test-preemption-trainjob-g9fzc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:09.800480369Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-cjgqr","name":"pod-dw-deployment-2sxqb-698d7dbcbf-9vkbl-bf644","reconcileID":"2cdcff67-5d78-445a-8f22-679f7d0c334e"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:09.848452733Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1276","msg":"Workload create event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-g9fzc-0f7bd","namespace":"test-ns-9p4tw"},"queue":"lq-d7bs9","status":"pending"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:09.848531396Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-9p4tw","name":"trainjob-test-preemption-trainjob-g9fzc-0f7bd","reconcileID":"8b6ebe37-626c-48f2-b763-eb0d8e6d2840"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:09.848583824Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","reconcileID":"a231de74-8bbb-4477-b09f-28556159dd9f","job":"test-ns-9p4tw/test-preemption-trainjob-g9fzc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"debug","ts":"2026-10-31T00:08:09.848562056Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"test-preemption-trainjob-g9fzc","namespace":"test-ns-9p4tw"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"CreatedWorkload","action":"CreatedWorkload","note":"Created Workload: test-ns-9p4tw/trainjob-test-preemption-trainjob-g9fzc-0f7bd"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:09.848603126Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-9p4tw","name":"trainjob-test-preemption-trainjob-g9fzc-0f7bd","reconcileID":"8b6ebe37-626c-48f2-b763-eb0d8e6d2840","isDeleted":false}
{"level":"Level(-2)","ts":"2026-10-31T00:08:09.848751794Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-9p4tw","name":"trainjob-test-preemption-trainjob-g9fzc-0f7bd","reconcileID":"477d3db1-d19f-4bfa-aeb9-57b70cab756c"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:09.848805546Z","logger":"scheduler","caller":"scheduler/scheduler.go:308","msg":"Obtained heads","replica-role":"leader","schedulingCycle":3,"headCount":1,"waitDuration":42.30960991}
{"level":"Level(-2)","ts":"2026-10-31T00:08:09.848835227Z","logger":"scheduler","caller":"scheduler/scheduler.go:323","msg":"Snapshot taken","replica-role":"leader","schedulingCycle":3,"duration":0.000011145}
{"level":"Level(-2)","ts":"2026-10-31T00:08:09.84889851Z","logger":"scheduler","caller":"scheduler/scheduler.go:328","msg":"Nomination done","replica-role":"leader","schedulingCycle":3,"entries":1,"inadmissibleEntries":0,"duration":0.000055945}
{"level":"Level(-2)","ts":"2026-10-31T00:08:09.848916676Z","logger":"scheduler","caller":"scheduler/scheduler.go:406","msg":"Attempting to schedule workload","replica-role":"leader","schedulingCycle":3,"workload":{"name":"trainjob-test-preemption-trainjob-g9fzc-0f7bd","namespace":"test-ns-9p4tw"},"clusterQueue":{"name":"cq-vfd72"}}
{"level":"Level(-2)","ts":"2026-10-31T00:08:09.848979288Z","logger":"scheduler","caller":"scheduler/scheduler.go:1041","msg":"Workload assumed in the cache","replica-role":"leader","schedulingCycle":3,"workload":{"name":"trainjob-test-preemption-trainjob-g9fzc-0f7bd","namespace":"test-ns-9p4tw"},"clusterQueue":{"name":"cq-vfd72"}}
{"level":"Level(-2)","ts":"2026-10-31T00:08:09.848997912Z","logger":"scheduler","caller":"scheduler/scheduler.go:362","msg":"Workload processing done","replica-role":"leader","schedulingCycle":3,"duration":0.000084718}
{"level":"Level(-2)","ts":"2026-10-31T00:08:09.849009609Z","logger":"scheduler","caller":"scheduler/scheduler.go:297","msg":"Scheduling cycle complete","replica-role":"leader","schedulingCycle":3,"duration":42.309814921}
{"level":"Level(-2)","ts":"2026-10-31T00:08:09.849021763Z","logger":"scheduler","caller":"scheduler/scheduler.go:295","msg":"Scheduling cycle starts","replica-role":"leader","schedulingCycle":4}
{"level":"Level(-2)","ts":"2026-10-31T00:08:09.959467343Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-9p4tw","name":"trainjob-test-preemption-trainjob-g9fzc-0f7bd","reconcileID":"eb106e0c-488b-4baf-9922-a24977101cbd"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:09.959541374Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-9p4tw","name":"trainjob-test-preemption-trainjob-g9fzc-0f7bd","reconcileID":"eb106e0c-488b-4baf-9922-a24977101cbd","isDeleted":false}
{"level":"Level(-2)","ts":"2026-10-31T00:08:09.959487863Z","logger":"scheduler","caller":"scheduler/scheduler.go:1000","msg":"Workload successfully admitted and assigned flavors","replica-role":"leader","schedulingCycle":3,"workload":{"name":"trainjob-test-preemption-trainjob-g9fzc-0f7bd","namespace":"test-ns-9p4tw"},"clusterQueue":{"name":"cq-vfd72"},"assignments":[{"name":"node","count":1}]}
{"level":"debug","ts":"2026-10-31T00:08:09.95949416Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-test-preemption-trainjob-g9fzc-0f7bd","namespace":"test-ns-9p4tw"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"QuotaReserved","action":"QuotaReserved","note":"Quota reserved in ClusterQueue cq-vfd72, wait time since queued was 1s"}
{"level":"debug","ts":"2026-10-31T00:08:09.959594393Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-test-preemption-trainjob-g9fzc-0f7bd","namespace":"test-ns-9p4tw"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"Admitted","action":"Admitted","note":"Admitted by ClusterQueue cq-vfd72, wait time since reservation was 1s"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:09.959458293Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-g9fzc-0f7bd","namespace":"test-ns-9p4tw"},"queue":"lq-d7bs9","status":"admitted","prevStatus":"pending","clusterQueue":"cq-vfd72"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:09.959749898Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-9p4tw","name":"trainjob-test-preemption-trainjob-g9fzc-0f7bd","reconcileID":"1d67a2c3-f022-44a6-bb47-5f22db1f2a0a"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:10.007094026Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","reconcileID":"74d9d9fd-821e-4c9e-ad85-88121332b472","job":"test-ns-9p4tw/test-preemption-trainjob-g9fzc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:10.165275686Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","reconcileID":"74d9d9fd-821e-4c9e-ad85-88121332b472","job":"test-ns-9p4tw/test-preemption-trainjob-g9fzc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-3)","ts":"2026-10-31T00:08:10.273590007Z","logger":"trainjob","caller":"jobframework/reconciler.go:594","msg":"Unsuspending job","replica-role":"leader","namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","reconcileID":"74d9d9fd-821e-4c9e-ad85-88121332b472","job":"test-ns-9p4tw/test-preemption-trainjob-g9fzc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-preemption-trainjob-g9fzc\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-3)","ts":"2026-10-31T00:08:10.273662302Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","reconcileID":"74d9d9fd-821e-4c9e-ad85-88121332b472","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-preemption-trainjob-g9fzc\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:10.273753837Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","reconcileID":"25a607f3-5169-4278-aa3a-f17c03475dad","job":"test-ns-9p4tw/test-preemption-trainjob-g9fzc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:10.431578875Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","reconcileID":"25a607f3-5169-4278-aa3a-f17c03475dad","job":"test-ns-9p4tw/test-preemption-trainjob-g9fzc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"info","ts":"2026-10-31T00:08:10.539785426Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-preemption-trainjob-g9fzc","namespace":"test-ns-9p4tw"},"namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"6473f31c-6b6c-4ea6-aae6-997e08afb4d7","replica-role":"leader"}
{"level":"info","ts":"2026-10-31T00:08:10.707275087Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-preemption-trainjob-g9fzc","namespace":"test-ns-9p4tw"},"namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"4d6934f0-2989-43fd-8eb4-c1c69eb374b9","replica-role":"leader"}
{"level":"debug","ts":"2026-10-31T00:08:10.762676797Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"test-preemption-trainjob-g9fzc","namespace":"test-ns-9p4tw"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"Started","action":"Started","note":"Admitted by clusterQueue cq-vfd72"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:10.762732158Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","reconcileID":"4cf99dd9-f765-476e-9c72-9be86ff56915","job":"test-ns-9p4tw/test-preemption-trainjob-g9fzc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:10.84932883Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-9p4tw","name":"lq-d7bs9","reconcileID":"064cd9f2-fae4-4d83-acbc-55006068292f"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:10.849343853Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-vfd72","reconcileID":"71a610ff-37bd-42f0-a621-ba410e7619c5"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:10.905856934Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-d7bs9","namespace":"test-ns-9p4tw"}}
{"level":"Level(-2)","ts":"2026-10-31T00:08:10.905989623Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-9p4tw","name":"lq-d7bs9","reconcileID":"fd933e5f-ed45-425e-b46d-c1a9be66402d"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:10.90865113Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-vfd72"}}
{"level":"Level(-2)","ts":"2026-10-31T00:08:10.908856407Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-vfd72","reconcileID":"cb63151f-6899-4f44-8153-c0ac3878d23c"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:10.921316274Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","reconcileID":"bee09278-4ce7-4f41-a828-314cf877082d","job":"test-ns-9p4tw/test-preemption-trainjob-g9fzc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:11.079823499Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","reconcileID":"dbfea7f6-bd46-49ca-9633-fa54f6f3a7db","job":"test-ns-9p4tw/test-preemption-trainjob-g9fzc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:11.238002372Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","reconcileID":"de5a6b47-a19e-45c6-861c-893f4a88a547","job":"test-ns-9p4tw/test-preemption-trainjob-g9fzc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:11.39660615Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","reconcileID":"b9c36f67-78db-4727-8402-a5dd27195817","job":"test-ns-9p4tw/test-preemption-trainjob-g9fzc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:12.110659081Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-9p4tw","name":"trainjob-test-preemption-trainjob-g9fzc-0f7bd","reconcileID":"2c8c1df8-83b0-44c5-b959-24b2f948be64"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:12.7098336Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-9p4tw","name":"trainjob-test-preemption-trainjob-g9fzc-0f7bd","reconcileID":"44b33f77-1ffe-404c-ae58-6f4c8bdc34e0"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:12.70989227Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-9p4tw","name":"trainjob-test-preemption-trainjob-g9fzc-0f7bd","reconcileID":"44b33f77-1ffe-404c-ae58-6f4c8bdc34e0","isDeleted":false}
{"level":"Level(-2)","ts":"2026-10-31T00:08:12.709835267Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-g9fzc-0f7bd","namespace":"test-ns-9p4tw"},"queue":"lq-d7bs9","status":"admitted","clusterQueue":"cq-vfd72"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:12.709929797Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1381","msg":"Workload will not be queued because the workload is not active","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-g9fzc-0f7bd","namespace":"test-ns-9p4tw"},"queue":"lq-d7bs9","status":"admitted","clusterQueue":"cq-vfd72"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:12.709960497Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","reconcileID":"5395173a-7216-47de-9618-5bde026d3802","job":"test-ns-9p4tw/test-preemption-trainjob-g9fzc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:12.710019172Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-9p4tw","name":"trainjob-test-preemption-trainjob-g9fzc-0f7bd","reconcileID":"d59bd272-882f-403c-a3f6-268386905dff"}
{"level":"debug","ts":"2026-10-31T00:08:12.822084674Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-test-preemption-trainjob-g9fzc-0f7bd","namespace":"test-ns-9p4tw"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"EvictedDueToDeactivated","action":"EvictedDueToDeactivated","note":"The workload is deactivated"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:12.822103836Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-g9fzc-0f7bd","namespace":"test-ns-9p4tw"},"queue":"lq-d7bs9","status":"admitted","clusterQueue":"cq-vfd72"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:12.822160234Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1381","msg":"Workload will not be queued because the workload is not active","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-g9fzc-0f7bd","namespace":"test-ns-9p4tw"},"queue":"lq-d7bs9","status":"admitted","clusterQueue":"cq-vfd72"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:12.822226769Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-9p4tw","name":"trainjob-test-preemption-trainjob-g9fzc-0f7bd","reconcileID":"43b7678b-404f-408a-b2e5-cf5cf883f693"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:12.822256995Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-9p4tw","name":"trainjob-test-preemption-trainjob-g9fzc-0f7bd","reconcileID":"43b02857-d9fa-4a6e-8c85-1f93f8cbf8a1"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:12.822298951Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-9p4tw","name":"trainjob-test-preemption-trainjob-g9fzc-0f7bd","reconcileID":"43b7678b-404f-408a-b2e5-cf5cf883f693","isDeleted":false}
{"level":"Level(-2)","ts":"2026-10-31T00:08:12.868575953Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","reconcileID":"014f0602-c623-4909-a77e-3a1f4c5418ca","job":"test-ns-9p4tw/test-preemption-trainjob-g9fzc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:13.112087971Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-9p4tw","name":"trainjob-test-preemption-trainjob-g9fzc-0f7bd","reconcileID":"78ddc1b0-2659-4b5c-bb58-74ee48318ad4"}
{"level":"info","ts":"2026-10-31T00:08:13.134771888Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-preemption-trainjob-g9fzc","namespace":"test-ns-9p4tw"},"namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"8b328c69-d05f-46d3-a55d-681cc6e945f0","replica-role":"leader"}
{"level":"error","ts":"2026-10-31T00:08:13.191210243Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","reconcileID":"014f0602-c623-4909-a77e-3a1f4c5418ca","error":"jobs are still active","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"debug","ts":"2026-10-31T00:08:13.191257296Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"test-preemption-trainjob-g9fzc","namespace":"test-ns-9p4tw"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"Stopped","action":"Stopped","note":"The workload is deactivated"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:13.191370374Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","reconcileID":"9e7104a4-81a9-4cb3-b86f-f62f78eaa664","job":"test-ns-9p4tw/test-preemption-trainjob-g9fzc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-10-31T00:08:13.34943977Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","reconcileID":"9e7104a4-81a9-4cb3-b86f-f62f78eaa664","error":"jobs are still active","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:13.349560871Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","reconcileID":"1eb2a41c-75ad-4508-8d98-a6580ee8894d","job":"test-ns-9p4tw/test-preemption-trainjob-g9fzc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-10-31T00:08:13.507835428Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","reconcileID":"1eb2a41c-75ad-4508-8d98-a6580ee8894d","error":"jobs are still active","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:13.507967009Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","reconcileID":"80686c58-c157-4de8-b9d8-29fb302a3881","job":"test-ns-9p4tw/test-preemption-trainjob-g9fzc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-10-31T00:08:13.665545715Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","reconcileID":"80686c58-c157-4de8-b9d8-29fb302a3881","error":"jobs are still active","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:13.665641706Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","reconcileID":"ea50e453-dad3-4d76-bd2b-aba5587b449a","job":"test-ns-9p4tw/test-preemption-trainjob-g9fzc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:13.711125298Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-9p4tw","name":"lq-d7bs9","reconcileID":"ae7a2370-dfb9-40c7-9969-ce103cd017b9"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:13.711117779Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-vfd72","reconcileID":"901334e6-6020-430b-8ada-e7513ebf8fe7"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:13.768637883Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-d7bs9","namespace":"test-ns-9p4tw"}}
{"level":"Level(-2)","ts":"2026-10-31T00:08:13.768812305Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-9p4tw","name":"lq-d7bs9","reconcileID":"2a0ab83b-37a9-4a97-9353-add9b45b4f13"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:13.771327273Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-vfd72"}}
{"level":"Level(-2)","ts":"2026-10-31T00:08:13.771470575Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-vfd72","reconcileID":"2d43afee-6d65-4d1c-9f68-4e0a71c16419"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:13.778905912Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-vfd72"}}
{"level":"Level(-2)","ts":"2026-10-31T00:08:13.778958658Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-vfd72","reconcileID":"42fea2d2-a5cd-4169-9b6b-4325f09f158c"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:13.779076503Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-9p4tw","name":"trainjob-test-preemption-trainjob-g9fzc-0f7bd","reconcileID":"eb1721d7-a148-4d2d-b69a-4950ab71e4ae"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:13.798502512Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:171","msg":"ResourceFlavor update event","replica-role":"leader","resourceFlavor":{"name":"rf-2gxjk"}}
{"level":"Level(-2)","ts":"2026-10-31T00:08:13.79857312Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-2gxjk","reconcileID":"1404a65d-4b28-44c9-b93e-f8daefdee113"}
{"level":"error","ts":"2026-10-31T00:08:13.822560375Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","reconcileID":"ea50e453-dad3-4d76-bd2b-aba5587b449a","error":"jobs are still active","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:13.822638069Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","reconcileID":"a0474d8f-d0a1-4f32-b4ac-6b9464923942","job":"test-ns-9p4tw/test-preemption-trainjob-g9fzc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:13.88769112Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:376","msg":"ClusterQueue delete event","replica-role":"leader","clusterQueue":{"name":"cq-vfd72"}}
{"level":"Level(-2)","ts":"2026-10-31T00:08:13.887758867Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-9p4tw","name":"lq-d7bs9","reconcileID":"4f49fdd4-ae5a-4b7f-93a2-ed9a0b5b3bbb"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:13.887763124Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:385","msg":"Cleared resource metrics for deleted ClusterQueue.","replica-role":"leader","clusterQueue":{"name":"cq-vfd72"}}
{"level":"Level(-2)","ts":"2026-10-31T00:08:13.887779281Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-9p4tw","name":"trainjob-test-preemption-trainjob-g9fzc-0f7bd","reconcileID":"fb6b6494-00c9-4340-b5d8-7e1b863b71c7"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:13.887865065Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-2gxjk","reconcileID":"29d3815a-9a3a-4593-b495-f5909b5ffec7"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:13.942951564Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-d7bs9","namespace":"test-ns-9p4tw"}}
{"level":"Level(-2)","ts":"2026-10-31T00:08:13.943071893Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-9p4tw","name":"lq-d7bs9","reconcileID":"2547e194-810d-437b-a001-af71391d39f1"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:13.997126523Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:159","msg":"ResourceFlavor delete event","replica-role":"leader","resourceFlavor":{"name":"rf-2gxjk"}}
{"level":"info","ts":"2026-10-31T00:08:14.088176699Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-preemption-trainjob-g9fzc","namespace":"test-ns-9p4tw"},"namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"e7baec80-60f6-4467-b1e2-c116300ba48a","replica-role":"leader"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:14.10807472Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:141","msg":"ResourceFlavor create event","replica-role":"leader","resourceFlavor":{"name":"rf-rwtrg"}}
{"level":"Level(-2)","ts":"2026-10-31T00:08:14.108172495Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-rwtrg","reconcileID":"c4edd04f-ceab-4f0d-97bc-465d7d48c777"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:14.229895284Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:351","msg":"ClusterQueue create event","replica-role":"leader","clusterQueue":{"name":"cq-5nszt"}}
{"level":"Level(-2)","ts":"2026-10-31T00:08:14.230088342Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-5nszt","reconcileID":"10410b5d-741e-4445-86ce-70d790d69b2b"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:14.250003131Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:240","msg":"LocalQueue create event","replica-role":"leader","localQueue":{"name":"lq-gnxsf","namespace":"test-ns-jjpdk"}}
{"level":"Level(-2)","ts":"2026-10-31T00:08:14.25010637Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-jjpdk","name":"lq-gnxsf","reconcileID":"a58183e5-eecb-4b72-aa4c-bee81f691b9c"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:14.256493909Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","reconcileID":"3f205e67-febe-4d7a-a8ef-4c8e1e8ab8c7","job":"test-ns-9p4tw/test-preemption-trainjob-g9fzc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:14.256566425Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-g9fzc-0f7bd","namespace":"test-ns-9p4tw"},"queue":"lq-d7bs9","status":"pending","prevStatus":"admitted","prevClusterQueue":"cq-vfd72"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:14.256586682Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-9p4tw","name":"trainjob-test-preemption-trainjob-g9fzc-0f7bd","reconcileID":"c70b90e3-daee-4f8c-a961-ef928a9b9002"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:14.256616983Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1381","msg":"Workload will not be queued because the workload is not active","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-g9fzc-0f7bd","namespace":"test-ns-9p4tw"},"queue":"lq-d7bs9","status":"pending","prevStatus":"admitted","prevClusterQueue":"cq-vfd72"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:14.25663086Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-9p4tw","name":"trainjob-test-preemption-trainjob-g9fzc-0f7bd","reconcileID":"c70b90e3-daee-4f8c-a961-ef928a9b9002","isDeleted":false}
{"level":"Level(-2)","ts":"2026-10-31T00:08:14.256691463Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-9p4tw","name":"trainjob-test-preemption-trainjob-g9fzc-0f7bd","reconcileID":"7ec67ab8-b2dd-434b-887b-18e738f87081"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:14.286198275Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-5nszt"}}
{"level":"Level(-2)","ts":"2026-10-31T00:08:14.286411001Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-5nszt","reconcileID":"c61ae8b2-c258-412e-a615-595a0d3ff5e8"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:14.307528504Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-gnxsf","namespace":"test-ns-jjpdk"}}
{"level":"Level(-2)","ts":"2026-10-31T00:08:14.307675535Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-jjpdk","name":"lq-gnxsf","reconcileID":"c63f55ad-0090-489a-a016-faa6b94cffe3"}
{"level":"error","ts":"2026-10-31T00:08:14.308863836Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","reconcileID":"3f205e67-febe-4d7a-a8ef-4c8e1e8ab8c7","job":"test-ns-9p4tw/test-preemption-trainjob-g9fzc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-g9fzc\" is forbidden: unable to create new content in namespace test-ns-9p4tw because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-10-31T00:08:14.308916381Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","reconcileID":"3f205e67-febe-4d7a-a8ef-4c8e1e8ab8c7","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-g9fzc\" is forbidden: unable to create new content in namespace test-ns-9p4tw because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:14.308978624Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","reconcileID":"2cf0fdbe-16b2-4bd5-9f6e-59d1ec800ea2","job":"test-ns-9p4tw/test-preemption-trainjob-g9fzc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-10-31T00:08:14.361220327Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","reconcileID":"2cf0fdbe-16b2-4bd5-9f6e-59d1ec800ea2","job":"test-ns-9p4tw/test-preemption-trainjob-g9fzc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-g9fzc\" is forbidden: unable to create new content in namespace test-ns-9p4tw because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-10-31T00:08:14.361315365Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","reconcileID":"2cf0fdbe-16b2-4bd5-9f6e-59d1ec800ea2","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-g9fzc\" is forbidden: unable to create new content in namespace test-ns-9p4tw because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:14.365344108Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-gnxsf","namespace":"test-ns-jjpdk"}}
{"level":"Level(-2)","ts":"2026-10-31T00:08:14.365359149Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-jjpdk","name":"lq-gnxsf","reconcileID":"2f09067e-ca15-4ca4-aa1b-b8cedf255bf3"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:14.365475068Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-jjpdk","name":"lq-gnxsf","reconcileID":"592ac2ca-e84c-4fff-80cd-fae344817cfd"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:14.368640433Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-g9fzc-0f7bd","namespace":"test-ns-9p4tw"},"queue":"lq-d7bs9","status":"pending"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:14.368693784Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1381","msg":"Workload will not be queued because the workload is not active","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-g9fzc-0f7bd","namespace":"test-ns-9p4tw"},"queue":"lq-d7bs9","status":"pending"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:14.36864736Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-9p4tw","name":"trainjob-test-preemption-trainjob-g9fzc-0f7bd","reconcileID":"d52146c3-33bc-4fc8-ab4f-b04fa4c0bcb3"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:14.368771738Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-9p4tw","name":"trainjob-test-preemption-trainjob-g9fzc-0f7bd","reconcileID":"b6ae60a5-82fb-4f24-b392-7bf1105d4733"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:14.36878579Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-9p4tw","name":"trainjob-test-preemption-trainjob-g9fzc-0f7bd","reconcileID":"d52146c3-33bc-4fc8-ab4f-b04fa4c0bcb3","isDeleted":false}
{"level":"Level(-2)","ts":"2026-10-31T00:08:14.368854225Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","reconcileID":"4432e3b7-ac0c-411e-ad38-559f8f5a5d61","job":"test-ns-9p4tw/test-preemption-trainjob-g9fzc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"info","ts":"2026-10-31T00:08:14.369542889Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:117","msg":"Validating create","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-runtimepatches-trainjob-rvmm2","namespace":"test-ns-jjpdk"},"namespace":"test-ns-jjpdk","name":"test-runtimepatches-trainjob-rvmm2","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:admin","requestID":"c0353e5f-a88b-4a11-95ce-de26eebb99cf","replica-role":"leader"}
{"level":"error","ts":"2026-10-31T00:08:14.421660079Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","reconcileID":"4432e3b7-ac0c-411e-ad38-559f8f5a5d61","job":"test-ns-9p4tw/test-preemption-trainjob-g9fzc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-g9fzc\" is forbidden: unable to create new content in namespace test-ns-9p4tw because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-10-31T00:08:14.421720305Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","reconcileID":"4432e3b7-ac0c-411e-ad38-559f8f5a5d61","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-g9fzc\" is forbidden: unable to create new content in namespace test-ns-9p4tw because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:14.42369368Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-jjpdk","name":"test-runtimepatches-trainjob-rvmm2","reconcileID":"7f5aa58b-30f7-4502-9a92-fa698172bcd1","job":"test-ns-jjpdk/test-runtimepatches-trainjob-rvmm2","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:14.743507306Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-jjpdk","name":"trainjob-test-runtimepatches-trainjob-rvmm2-63734","reconcileID":"f5850085-36c4-41c2-9cdf-258f8b07c1d3"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:14.743515403Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1276","msg":"Workload create event","replica-role":"leader","workload":{"name":"trainjob-test-runtimepatches-trainjob-rvmm2-63734","namespace":"test-ns-jjpdk"},"queue":"lq-gnxsf","status":"pending"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:14.74355094Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-jjpdk","name":"trainjob-test-runtimepatches-trainjob-rvmm2-63734","reconcileID":"f5850085-36c4-41c2-9cdf-258f8b07c1d3","isDeleted":false}
{"level":"debug","ts":"2026-10-31T00:08:14.743646781Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"test-runtimepatches-trainjob-rvmm2","namespace":"test-ns-jjpdk"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"CreatedWorkload","action":"CreatedWorkload","note":"Created Workload: test-ns-jjpdk/trainjob-test-runtimepatches-trainjob-rvmm2-63734"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:14.743680317Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","reconcileID":"47e76473-186b-40f4-a847-22ccd4e7af99","job":"test-ns-9p4tw/test-preemption-trainjob-g9fzc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:14.743703486Z","logger":"scheduler","caller":"scheduler/scheduler.go:308","msg":"Obtained heads","replica-role":"leader","schedulingCycle":4,"headCount":1,"waitDuration":4.894680732}
{"level":"Level(-2)","ts":"2026-10-31T00:08:14.74372241Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-jjpdk","name":"trainjob-test-runtimepatches-trainjob-rvmm2-63734","reconcileID":"7b8fbd46-4edc-4197-9a21-207ebfddd7f2"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:14.743734884Z","logger":"scheduler","caller":"scheduler/scheduler.go:323","msg":"Snapshot taken","replica-role":"leader","schedulingCycle":4,"duration":0.000010808}
{"level":"Level(-2)","ts":"2026-10-31T00:08:14.743803915Z","logger":"scheduler","caller":"scheduler/scheduler.go:328","msg":"Nomination done","replica-role":"leader","schedulingCycle":4,"entries":1,"inadmissibleEntries":0,"duration":0.000061086}
{"level":"Level(-2)","ts":"2026-10-31T00:08:14.743823377Z","logger":"scheduler","caller":"scheduler/scheduler.go:406","msg":"Attempting to schedule workload","replica-role":"leader","schedulingCycle":4,"workload":{"name":"trainjob-test-runtimepatches-trainjob-rvmm2-63734","namespace":"test-ns-jjpdk"},"clusterQueue":{"name":"cq-5nszt"}}
{"level":"Level(-2)","ts":"2026-10-31T00:08:14.743899142Z","logger":"scheduler","caller":"scheduler/scheduler.go:1041","msg":"Workload assumed in the cache","replica-role":"leader","schedulingCycle":4,"workload":{"name":"trainjob-test-runtimepatches-trainjob-rvmm2-63734","namespace":"test-ns-jjpdk"},"clusterQueue":{"name":"cq-5nszt"}}
{"level":"Level(-2)","ts":"2026-10-31T00:08:14.743918346Z","logger":"scheduler","caller":"scheduler/scheduler.go:362","msg":"Workload processing done","replica-role":"leader","schedulingCycle":4,"duration":0.000100583}
{"level":"Level(-2)","ts":"2026-10-31T00:08:14.743932688Z","logger":"scheduler","caller":"scheduler/scheduler.go:297","msg":"Scheduling cycle complete","replica-role":"leader","schedulingCycle":4,"duration":4.894910746}
{"level":"Level(-2)","ts":"2026-10-31T00:08:14.743944539Z","logger":"scheduler","caller":"scheduler/scheduler.go:295","msg":"Scheduling cycle starts","replica-role":"leader","schedulingCycle":5}
{"level":"error","ts":"2026-10-31T00:08:14.796951247Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","reconcileID":"47e76473-186b-40f4-a847-22ccd4e7af99","job":"test-ns-9p4tw/test-preemption-trainjob-g9fzc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-g9fzc\" is forbidden: unable to create new content in namespace test-ns-9p4tw because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-10-31T00:08:14.797016837Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","reconcileID":"47e76473-186b-40f4-a847-22ccd4e7af99","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-g9fzc\" is forbidden: unable to create new content in namespace test-ns-9p4tw because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:14.797106846Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-jjpdk","name":"test-runtimepatches-trainjob-rvmm2","reconcileID":"94703c57-eacd-4c77-ae28-01f2bf4a7cef","job":"test-ns-jjpdk/test-runtimepatches-trainjob-rvmm2","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:14.857065467Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-runtimepatches-trainjob-rvmm2-63734","namespace":"test-ns-jjpdk"},"queue":"lq-gnxsf","status":"admitted","prevStatus":"pending","clusterQueue":"cq-5nszt"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:14.857148136Z","logger":"scheduler","caller":"scheduler/scheduler.go:1000","msg":"Workload successfully admitted and assigned flavors","replica-role":"leader","schedulingCycle":4,"workload":{"name":"trainjob-test-runtimepatches-trainjob-rvmm2-63734","namespace":"test-ns-jjpdk"},"clusterQueue":{"name":"cq-5nszt"},"assignments":[{"name":"node","flavors":{"cpu":"rf-rwtrg","memory":"rf-rwtrg"},"resourceUsage":{"cpu":"100m","memory":"128Mi"},"count":1}]}
{"level":"debug","ts":"2026-10-31T00:08:14.857179595Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-test-runtimepatches-trainjob-rvmm2-63734","namespace":"test-ns-jjpdk"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"QuotaReserved","action":"QuotaReserved","note":"Quota reserved in ClusterQueue cq-5nszt, wait time since queued was 1s"}
{"level":"debug","ts":"2026-10-31T00:08:14.857200445Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-test-runtimepatches-trainjob-rvmm2-63734","namespace":"test-ns-jjpdk"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"Admitted","action":"Admitted","note":"Admitted by ClusterQueue cq-5nszt, wait time since reservation was 1s"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:14.857065696Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-jjpdk","name":"trainjob-test-runtimepatches-trainjob-rvmm2-63734","reconcileID":"26530f6d-5aaa-40e4-8655-1ff6cfa06fd0"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:14.857306648Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-jjpdk","name":"trainjob-test-runtimepatches-trainjob-rvmm2-63734","reconcileID":"eef24d79-9771-42b7-bdea-5de0b45b842a"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:14.857318154Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-jjpdk","name":"trainjob-test-runtimepatches-trainjob-rvmm2-63734","reconcileID":"26530f6d-5aaa-40e4-8655-1ff6cfa06fd0","isDeleted":false}
{"level":"Level(-2)","ts":"2026-10-31T00:08:14.955595636Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-jjpdk","name":"test-runtimepatches-trainjob-rvmm2","reconcileID":"b2447566-267a-4a8b-a093-04aa3871fb60","job":"test-ns-jjpdk/test-runtimepatches-trainjob-rvmm2","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:15.112597123Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-jjpdk","name":"test-runtimepatches-trainjob-rvmm2","reconcileID":"b2447566-267a-4a8b-a093-04aa3871fb60","job":"test-ns-jjpdk/test-runtimepatches-trainjob-rvmm2","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-3)","ts":"2026-10-31T00:08:15.219527535Z","logger":"trainjob","caller":"jobframework/reconciler.go:594","msg":"Unsuspending job","replica-role":"leader","namespace":"test-ns-jjpdk","name":"test-runtimepatches-trainjob-rvmm2","reconcileID":"b2447566-267a-4a8b-a093-04aa3871fb60","job":"test-ns-jjpdk/test-runtimepatches-trainjob-rvmm2","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-runtimepatches-trainjob-rvmm2\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-3)","ts":"2026-10-31T00:08:15.219610108Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-jjpdk","name":"test-runtimepatches-trainjob-rvmm2","reconcileID":"b2447566-267a-4a8b-a093-04aa3871fb60","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-runtimepatches-trainjob-rvmm2\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:15.219721598Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","reconcileID":"bca5d6b9-4151-4eb7-935d-62e15356d79e","job":"test-ns-9p4tw/test-preemption-trainjob-g9fzc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:15.257771011Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-9p4tw","name":"lq-d7bs9","reconcileID":"0cd08a94-209a-4715-ae5b-af1507394501"}
{"level":"error","ts":"2026-10-31T00:08:15.272125273Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","reconcileID":"bca5d6b9-4151-4eb7-935d-62e15356d79e","job":"test-ns-9p4tw/test-preemption-trainjob-g9fzc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-g9fzc\" is forbidden: unable to create new content in namespace test-ns-9p4tw because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-10-31T00:08:15.272197486Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","reconcileID":"bca5d6b9-4151-4eb7-935d-62e15356d79e","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-g9fzc\" is forbidden: unable to create new content in namespace test-ns-9p4tw because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:15.272276642Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-jjpdk","name":"test-runtimepatches-trainjob-rvmm2","reconcileID":"715f5214-9bd0-4d49-90ec-d3e07ebc2078","job":"test-ns-jjpdk/test-runtimepatches-trainjob-rvmm2","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:15.430647005Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-jjpdk","name":"test-runtimepatches-trainjob-rvmm2","reconcileID":"715f5214-9bd0-4d49-90ec-d3e07ebc2078","job":"test-ns-jjpdk/test-runtimepatches-trainjob-rvmm2","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:15.51236122Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-5nszt"}}
{"level":"Level(-2)","ts":"2026-10-31T00:08:15.512451253Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-5nszt","reconcileID":"43d7df77-fc9b-4e33-afe5-6b428a2fcf79"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:15.51249799Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-jjpdk","name":"trainjob-test-runtimepatches-trainjob-rvmm2-63734","reconcileID":"d7d43a8a-1585-437a-92fd-a0d33bd101ac"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:15.530399822Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:171","msg":"ResourceFlavor update event","replica-role":"leader","resourceFlavor":{"name":"rf-rwtrg"}}
{"level":"Level(-2)","ts":"2026-10-31T00:08:15.530464566Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-rwtrg","reconcileID":"6b405395-bffa-4673-ac4e-2cfab7cd2cbc"}
{"level":"info","ts":"2026-10-31T00:08:15.539382579Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-runtimepatches-trainjob-rvmm2","namespace":"test-ns-jjpdk"},"namespace":"test-ns-jjpdk","name":"test-runtimepatches-trainjob-rvmm2","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"9fb1d2fd-4205-44a3-b156-bb1e2fc11e4a","replica-role":"leader"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:15.570685192Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-5nszt"}}
{"level":"Level(-2)","ts":"2026-10-31T00:08:15.570744447Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-5nszt","reconcileID":"603ae434-2ac4-492e-ab2f-4e88ef022043"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:15.570822845Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-jjpdk","name":"trainjob-test-runtimepatches-trainjob-rvmm2-63734","reconcileID":"a182156d-3040-4a70-afd5-9d65482135d1"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:15.570821687Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-jjpdk","name":"lq-gnxsf","reconcileID":"e7423193-0953-4e63-b2d1-6af176d67c1f"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:15.62902794Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-gnxsf","namespace":"test-ns-jjpdk"}}
{"level":"Level(-2)","ts":"2026-10-31T00:08:15.629156036Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-jjpdk","name":"lq-gnxsf","reconcileID":"9bc27eeb-62db-45e8-a192-a7b733596b7c"}
{"level":"info","ts":"2026-10-31T00:08:15.704944543Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-runtimepatches-trainjob-rvmm2","namespace":"test-ns-jjpdk"},"namespace":"test-ns-jjpdk","name":"test-runtimepatches-trainjob-rvmm2","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"b1cc975f-0f49-44c5-a462-f5ca44539360","replica-role":"leader"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:15.763620848Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","reconcileID":"4b3e4554-1cff-4275-9ed0-8966a659ab13","job":"test-ns-9p4tw/test-preemption-trainjob-g9fzc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"debug","ts":"2026-10-31T00:08:15.764084128Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"test-runtimepatches-trainjob-rvmm2","namespace":"test-ns-jjpdk"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"Started","action":"Started","note":"Admitted by clusterQueue cq-5nszt"}
{"level":"error","ts":"2026-10-31T00:08:15.815938068Z","logger":"events","caller":"events/event_broadcaster.go:270","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-jjpdk","name":"test-runtimepatches-trainjob-rvmm2.18e3749d584a018e"},"error":"events.events.k8s.io \"test-runtimepatches-trainjob-rvmm2.18e3749d584a018e\" is forbidden: unable to create new content in namespace test-ns-jjpdk because it is being terminated","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/workspace/upstream/kueue/src/vendor/k8s.io/client-go/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).attemptRecording\n\t/workspace/upstream/kueue/src/vendor/k8s.io/client-go/tools/events/event_broadcaster.go:211\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).recordToSink.func1\n\t/workspace/upstream/kueue/src/vendor/k8s.io/client-go/tools/events/event_broadcaster.go:200"}
{"level":"error","ts":"2026-10-31T00:08:15.816235104Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","reconcileID":"4b3e4554-1cff-4275-9ed0-8966a659ab13","job":"test-ns-9p4tw/test-preemption-trainjob-g9fzc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-g9fzc\" is forbidden: unable to create new content in namespace test-ns-9p4tw because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-10-31T00:08:15.816319788Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","reconcileID":"4b3e4554-1cff-4275-9ed0-8966a659ab13","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-g9fzc\" is forbidden: unable to create new content in namespace test-ns-9p4tw because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:15.816431138Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-jjpdk","name":"test-runtimepatches-trainjob-rvmm2","reconcileID":"8cd00835-0e81-4190-a99f-afbb644461d8","job":"test-ns-jjpdk/test-runtimepatches-trainjob-rvmm2","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-10-31T00:08:15.869091209Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-jjpdk","name":"test-runtimepatches-trainjob-rvmm2","reconcileID":"8cd00835-0e81-4190-a99f-afbb644461d8","job":"test-ns-jjpdk/test-runtimepatches-trainjob-rvmm2","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-runtimepatches-trainjob-rvmm2\" is forbidden: unable to create new content in namespace test-ns-jjpdk because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-10-31T00:08:15.869153664Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-jjpdk","name":"test-runtimepatches-trainjob-rvmm2","reconcileID":"8cd00835-0e81-4190-a99f-afbb644461d8","error":"workloads.kueue.x-k8s.io \"test-runtimepatches-trainjob-rvmm2\" is forbidden: unable to create new content in namespace test-ns-jjpdk because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:15.87447716Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-jjpdk","name":"test-runtimepatches-trainjob-rvmm2","reconcileID":"b0bb10ce-f603-4392-bd41-dba6e2ce67b2","job":"test-ns-jjpdk/test-runtimepatches-trainjob-rvmm2","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-10-31T00:08:15.927462472Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-jjpdk","name":"test-runtimepatches-trainjob-rvmm2","reconcileID":"b0bb10ce-f603-4392-bd41-dba6e2ce67b2","job":"test-ns-jjpdk/test-runtimepatches-trainjob-rvmm2","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-runtimepatches-trainjob-rvmm2\" is forbidden: unable to create new content in namespace test-ns-jjpdk because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-10-31T00:08:15.927529708Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-jjpdk","name":"test-runtimepatches-trainjob-rvmm2","reconcileID":"b0bb10ce-f603-4392-bd41-dba6e2ce67b2","error":"workloads.kueue.x-k8s.io \"test-runtimepatches-trainjob-rvmm2\" is forbidden: unable to create new content in namespace test-ns-jjpdk because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:15.927632774Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-jjpdk","name":"test-runtimepatches-trainjob-rvmm2","reconcileID":"4885eb5c-e094-4a3a-bde1-d31187206e86","job":"test-ns-jjpdk/test-runtimepatches-trainjob-rvmm2","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-10-31T00:08:15.979912696Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-jjpdk","name":"test-runtimepatches-trainjob-rvmm2","reconcileID":"4885eb5c-e094-4a3a-bde1-d31187206e86","job":"test-ns-jjpdk/test-runtimepatches-trainjob-rvmm2","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-runtimepatches-trainjob-rvmm2\" is forbidden: unable to create new content in namespace test-ns-jjpdk because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-10-31T00:08:15.979981146Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-jjpdk","name":"test-runtimepatches-trainjob-rvmm2","reconcileID":"4885eb5c-e094-4a3a-bde1-d31187206e86","error":"workloads.kueue.x-k8s.io \"test-runtimepatches-trainjob-rvmm2\" is forbidden: unable to create new content in namespace test-ns-jjpdk because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:15.980070232Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","reconcileID":"adeb61a0-7313-4002-8062-1b1905b17a31","job":"test-ns-9p4tw/test-preemption-trainjob-g9fzc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-10-31T00:08:16.032359121Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","reconcileID":"adeb61a0-7313-4002-8062-1b1905b17a31","job":"test-ns-9p4tw/test-preemption-trainjob-g9fzc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-g9fzc\" is forbidden: unable to create new content in namespace test-ns-9p4tw because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-10-31T00:08:16.032421912Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-9p4tw","name":"test-preemption-trainjob-g9fzc","reconcileID":"adeb61a0-7313-4002-8062-1b1905b17a31","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-g9fzc\" is forbidden: unable to create new content in namespace test-ns-9p4tw because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:16.032509182Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-jjpdk","name":"test-runtimepatches-trainjob-rvmm2","reconcileID":"3e1e1c2e-432b-4607-a8a5-693ddc775250","job":"test-ns-jjpdk/test-runtimepatches-trainjob-rvmm2","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-10-31T00:08:16.085028416Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-jjpdk","name":"test-runtimepatches-trainjob-rvmm2","reconcileID":"3e1e1c2e-432b-4607-a8a5-693ddc775250","job":"test-ns-jjpdk/test-runtimepatches-trainjob-rvmm2","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-runtimepatches-trainjob-rvmm2\" is forbidden: unable to create new content in namespace test-ns-jjpdk because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-10-31T00:08:16.085092148Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-jjpdk","name":"test-runtimepatches-trainjob-rvmm2","reconcileID":"3e1e1c2e-432b-4607-a8a5-693ddc775250","error":"workloads.kueue.x-k8s.io \"test-runtimepatches-trainjob-rvmm2\" is forbidden: unable to create new content in namespace test-ns-jjpdk because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:16.085193594Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-jjpdk","name":"test-runtimepatches-trainjob-rvmm2","reconcileID":"580df4c9-6b38-418b-a06f-8458c9f76035","job":"test-ns-jjpdk/test-runtimepatches-trainjob-rvmm2","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-10-31T00:08:16.137460003Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-jjpdk","name":"test-runtimepatches-trainjob-rvmm2","reconcileID":"580df4c9-6b38-418b-a06f-8458c9f76035","job":"test-ns-jjpdk/test-runtimepatches-trainjob-rvmm2","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-runtimepatches-trainjob-rvmm2\" is forbidden: unable to create new content in namespace test-ns-jjpdk because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-10-31T00:08:16.137520573Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-jjpdk","name":"test-runtimepatches-trainjob-rvmm2","reconcileID":"580df4c9-6b38-418b-a06f-8458c9f76035","error":"workloads.kueue.x-k8s.io \"test-runtimepatches-trainjob-rvmm2\" is forbidden: unable to create new content in namespace test-ns-jjpdk because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-10-31T00:08:16.141082032Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-jjpdk","name":"test-runtimepatches-trainjob-rvmm2","reconcileID":"0504a0eb-6002-47b8-b0a9-79c6accb437a","job":"test-ns-jjpdk/test-runtimepatches-trainjob-rvmm2","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}