{"level":"Level(-2)","ts":"2026-11-03T03:47:41.241250887Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-n4x8n","name":"e2e-test-runtime-update","reconcileID":"ba187c88-3469-4383-bc50-29475c654987","job":"e2e-n4x8n/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-n4x8n"}
{"level":"Level(-2)","ts":"2026-11-03T03:47:41.260360732Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-n4x8n","name":"e2e-test-runtime-update","reconcileID":"234c48c3-6af0-453f-aac2-4e1178d65d35","job":"e2e-n4x8n/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-n4x8n"}
{"level":"Level(-2)","ts":"2026-11-03T03:47:41.353585838Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-n4x8n","name":"e2e-test-runtime-update","reconcileID":"e917b4a3-b76c-487e-92d8-2bbca637a512","job":"e2e-n4x8n/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-n4x8n"}
{"level":"Level(-2)","ts":"2026-11-03T03:47:41.681280486Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-n4x8n","name":"e2e-test-runtime-update","reconcileID":"870c312a-6f44-4a27-ae1b-aac3b3754b79","job":"e2e-n4x8n/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-n4x8n"}
{"level":"Level(-2)","ts":"2026-11-03T03:47:41.793466198Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-n4x8n","name":"e2e-test-runtime-update","reconcileID":"87a2fb3f-c67c-48df-a10b-f7da1f39ca23","job":"e2e-n4x8n/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-n4x8n"}
{"level":"Level(-2)","ts":"2026-11-03T03:47:41.903363179Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-n4x8n","name":"e2e-test-runtime-update","reconcileID":"08445573-5f36-4be2-8495-bf31b53c6439","job":"e2e-n4x8n/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-n4x8n"}
{"level":"Level(-2)","ts":"2026-11-03T03:47:41.914119046Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-n4x8n","name":"e2e-test-runtime-update","reconcileID":"286cb6d1-0bfb-4efa-b5b7-2c66f992a086","job":"e2e-n4x8n/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-n4x8n"}
{"level":"Level(-2)","ts":"2026-11-03T03:47:41.997219662Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-n4x8n","name":"e2e-test-runtime-update","reconcileID":"4f7b98c8-c28f-48e0-b50a-6b7446a03f31","job":"e2e-n4x8n/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-n4x8n"}
{"level":"Level(-2)","ts":"2026-11-03T03:47:42.188559741Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-n4x8n","name":"e2e-test-runtime-update","reconcileID":"362a15dc-3a22-4df2-b303-f972326af5c2","job":"e2e-n4x8n/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-n4x8n"}
{"level":"Level(-2)","ts":"2026-11-03T03:47:42.458242701Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-n4x8n","name":"e2e-test-runtime-update","reconcileID":"11f66a77-a1e9-44a2-a323-0b3c117507d0","job":"e2e-n4x8n/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-n4x8n"}
{"level":"Level(-2)","ts":"2026-11-03T03:47:42.62972883Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-n4x8n","name":"e2e-test-runtime-update","reconcileID":"743d8540-1243-4b64-ad1f-363fdd0ae9ad","job":"e2e-n4x8n/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-n4x8n"}
{"level":"Level(-2)","ts":"2026-11-03T03:47:42.739594654Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-n4x8n","name":"e2e-test-runtime-update","reconcileID":"ddf521b9-4fae-4265-8c4c-e4a933d928da","job":"e2e-n4x8n/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-n4x8n"}
{"level":"Level(-2)","ts":"2026-11-03T03:47:42.807326474Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-n4x8n","name":"e2e-test-runtime-update","reconcileID":"257c70a0-f4b0-4bc7-bdf7-57729f795ba8","job":"e2e-n4x8n/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-n4x8n"}
{"level":"Level(-2)","ts":"2026-11-03T03:47:42.868492773Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-n4x8n","name":"e2e-test-runtime-update","reconcileID":"027198d4-58ec-4e31-ae99-abf3717f8622","job":"e2e-n4x8n/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-n4x8n"}
{"level":"Level(-2)","ts":"2026-11-03T03:47:42.935257616Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-n4x8n","name":"e2e-test-runtime-update","reconcileID":"7ac128b8-0dc2-4f6a-a257-bfe712916703","job":"e2e-n4x8n/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-n4x8n"}
{"level":"Level(-2)","ts":"2026-11-03T03:47:42.961618555Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-n4x8n","name":"e2e-test-runtime-update","reconcileID":"0443d8e9-f366-44dc-952d-bb27b03e0df2","job":"e2e-n4x8n/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-n4x8n"}
{"level":"Level(-2)","ts":"2026-11-03T03:47:43.071857383Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-n4x8n","name":"e2e-test-runtime-update","reconcileID":"d612a647-7dc4-4be6-a6a8-26a7bcd9739f","job":"e2e-n4x8n/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-n4x8n"}
{"level":"Level(-2)","ts":"2026-11-03T03:48:39.09477409Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-7d6sw","name":"test-dra-trainjob-slctj","reconcileID":"d9d55871-2b41-4b14-ae96-60fe42a292c2","job":"test-ns-7d6sw/test-dra-trainjob-slctj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-7d6sw"}
{"level":"Level(-2)","ts":"2026-11-03T03:48:39.342255596Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-7d6sw","name":"test-dra-trainjob-slctj","reconcileID":"fb0a19ed-6a7f-43fc-993d-53ee7b96e811","job":"test-ns-7d6sw/test-dra-trainjob-slctj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-7d6sw"}
{"level":"Level(-2)","ts":"2026-11-03T03:48:39.569903758Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-7d6sw","name":"test-dra-trainjob-slctj","reconcileID":"3b8a57e8-140f-4156-bdff-59eaa2f220b9","job":"test-ns-7d6sw/test-dra-trainjob-slctj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-7d6sw"}
{"level":"Level(-2)","ts":"2026-11-03T03:48:39.68256312Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-7d6sw","name":"test-dra-trainjob-slctj","reconcileID":"32cb0a71-3de1-482b-8211-0b9d9d0865e5","job":"test-ns-7d6sw/test-dra-trainjob-slctj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-7d6sw"}
{"level":"Level(-2)","ts":"2026-11-03T03:48:39.766150611Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-7d6sw","name":"test-dra-trainjob-slctj","reconcileID":"203ffaf3-6f9a-47e2-affd-b1130a868616","job":"test-ns-7d6sw/test-dra-trainjob-slctj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-7d6sw"}
{"level":"Level(-2)","ts":"2026-11-03T03:48:39.879029712Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-7d6sw","name":"test-dra-trainjob-slctj","reconcileID":"821652d6-20db-4ac4-a166-94ec5e6c8062","job":"test-ns-7d6sw/test-dra-trainjob-slctj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-7d6sw"}
{"level":"Level(-2)","ts":"2026-11-03T03:48:43.776452091Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-sw8lc","name":"test-trainjob-xlwwg","reconcileID":"c8c762e9-d0ee-4b74-a5fe-c2fa0d5d2c09","job":"test-ns-sw8lc/test-trainjob-xlwwg","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-sw8lc"}
{"level":"Level(-2)","ts":"2026-11-03T03:48:43.9989832Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-sw8lc","name":"test-trainjob-xlwwg","reconcileID":"a60d24e5-9f81-4373-8e95-fc5cb69a76d4","job":"test-ns-sw8lc/test-trainjob-xlwwg","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-sw8lc"}
{"level":"Level(-2)","ts":"2026-11-03T03:48:44.225287913Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-sw8lc","name":"test-trainjob-xlwwg","reconcileID":"e8482d9f-ce2a-44ad-ab01-895cc39800c7","job":"test-ns-sw8lc/test-trainjob-xlwwg","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-sw8lc"}
{"level":"Level(-2)","ts":"2026-11-03T03:48:44.335529206Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-sw8lc","name":"test-trainjob-xlwwg","reconcileID":"20d9e199-780b-486e-9521-24278bbbaeef","job":"test-ns-sw8lc/test-trainjob-xlwwg","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-sw8lc"}
{"level":"Level(-2)","ts":"2026-11-03T03:48:44.341990338Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-sw8lc","name":"test-trainjob-xlwwg","reconcileID":"57e0cdc6-3aba-4e91-b690-ce71aa920012","job":"test-ns-sw8lc/test-trainjob-xlwwg","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-sw8lc"}
{"level":"Level(-2)","ts":"2026-11-03T03:48:44.448241353Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-sw8lc","name":"test-trainjob-xlwwg","reconcileID":"d00daada-0bc6-4616-9688-be81e39b7e71","job":"test-ns-sw8lc/test-trainjob-xlwwg","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-sw8lc"}
{"level":"Level(-2)","ts":"2026-11-03T03:48:46.395486611Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-7d6sw","name":"test-dra-trainjob-slctj","reconcileID":"12004f96-ae0a-434e-88d4-1a810e60b6e6","job":"test-ns-7d6sw/test-dra-trainjob-slctj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-7d6sw"}
{"level":"Level(-2)","ts":"2026-11-03T03:48:46.508404588Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-7d6sw","name":"test-dra-trainjob-slctj","reconcileID":"469a899b-dfac-4e0f-b6b3-a5045cb4566d","job":"test-ns-7d6sw/test-dra-trainjob-slctj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-7d6sw"}
{"level":"Level(-2)","ts":"2026-11-03T03:48:46.826347556Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-sw8lc","name":"test-trainjob-xlwwg","reconcileID":"ff034a06-60e8-410b-a7a5-92831f6cd955","job":"test-ns-sw8lc/test-trainjob-xlwwg","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-sw8lc"}
{"level":"Level(-2)","ts":"2026-11-03T03:48:46.939099687Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-sw8lc","name":"test-trainjob-xlwwg","reconcileID":"e5869765-8f8b-44c2-bfa1-4c74c5f78aee","job":"test-ns-sw8lc/test-trainjob-xlwwg","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-sw8lc"}
{"level":"Level(-2)","ts":"2026-11-03T03:48:53.863488802Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-sw8lc","name":"test-trainjob-xlwwg","reconcileID":"1bc621b0-8a97-4ce7-96eb-60f6df046001","job":"test-ns-sw8lc/test-trainjob-xlwwg","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-sw8lc"}
{"level":"Level(-2)","ts":"2026-11-03T03:48:53.97556187Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-sw8lc","name":"test-trainjob-xlwwg","reconcileID":"a9ce85ac-96db-4139-b5a0-75eaabb6f35d","job":"test-ns-sw8lc/test-trainjob-xlwwg","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-sw8lc"}
{"level":"Level(-2)","ts":"2026-11-03T03:48:54.872384291Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-sw8lc","name":"test-trainjob-xlwwg","reconcileID":"2c63dfdf-1aca-4ea0-bd19-95c26344ad22","job":"test-ns-sw8lc/test-trainjob-xlwwg","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-sw8lc"}
{"level":"Level(-2)","ts":"2026-11-03T03:48:54.983839793Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-sw8lc","name":"test-trainjob-xlwwg","reconcileID":"4813dc28-d009-40d4-8285-2d9a4e01717f","job":"test-ns-sw8lc/test-trainjob-xlwwg","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-sw8lc"}
{"level":"Level(-2)","ts":"2026-11-03T03:48:54.996532478Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-sw8lc","name":"test-trainjob-xlwwg","reconcileID":"077a63d8-2eb5-4ffc-b2e1-f49e74ab679d","job":"test-ns-sw8lc/test-trainjob-xlwwg","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-sw8lc"}
{"level":"Level(-2)","ts":"2026-11-03T03:48:55.095388645Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-sw8lc","name":"test-trainjob-xlwwg","reconcileID":"68ca144c-24c3-478e-be77-b4fc97c4c858","job":"test-ns-sw8lc/test-trainjob-xlwwg","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-sw8lc"}
{"level":"Level(-2)","ts":"2026-11-03T03:48:56.168731046Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jsqs6","name":"test-trainjob-zx88l","reconcileID":"2d976084-ebaa-4187-8c43-a532a233124d","job":"test-ns-jsqs6/test-trainjob-zx88l","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jsqs6"}
{"level":"Level(-2)","ts":"2026-11-03T03:48:56.403268887Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jsqs6","name":"test-trainjob-zx88l","reconcileID":"7ef146bc-b4f4-44e5-a76b-6235df68293e","job":"test-ns-jsqs6/test-trainjob-zx88l","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jsqs6"}
{"level":"Level(-2)","ts":"2026-11-03T03:48:56.6302465Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jsqs6","name":"test-trainjob-zx88l","reconcileID":"806efba7-207a-492b-980c-9c795daffacc","job":"test-ns-jsqs6/test-trainjob-zx88l","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jsqs6"}
{"level":"Level(-2)","ts":"2026-11-03T03:48:56.740732275Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jsqs6","name":"test-trainjob-zx88l","reconcileID":"96e2cbb6-1268-416b-a4da-13c8c4ae69f8","job":"test-ns-jsqs6/test-trainjob-zx88l","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jsqs6"}
{"level":"Level(-2)","ts":"2026-11-03T03:48:56.746473986Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jsqs6","name":"test-trainjob-zx88l","reconcileID":"0f623589-7e5e-4637-be9b-a205e602472a","job":"test-ns-jsqs6/test-trainjob-zx88l","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jsqs6"}
{"level":"Level(-2)","ts":"2026-11-03T03:48:56.853297701Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jsqs6","name":"test-trainjob-zx88l","reconcileID":"a3135fb7-0bae-4c16-a942-18ba03ad8704","job":"test-ns-jsqs6/test-trainjob-zx88l","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jsqs6"}
{"level":"Level(-2)","ts":"2026-11-03T04:00:10.231257176Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jsqs6","name":"test-trainjob-zx88l","reconcileID":"1ad188d3-81da-45fd-9e07-ba7d7d66872b","job":"test-ns-jsqs6/test-trainjob-zx88l","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jsqs6"}
{"level":"Level(-2)","ts":"2026-11-03T04:00:10.342571927Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jsqs6","name":"test-trainjob-zx88l","reconcileID":"87b656d8-2cd0-4531-9bd2-2937fe57a74f","job":"test-ns-jsqs6/test-trainjob-zx88l","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jsqs6"}
{"level":"Level(-2)","ts":"2026-11-03T04:00:15.194546152Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jsqs6","name":"test-trainjob-zx88l","reconcileID":"8a35f099-0230-4c0a-b51a-c9f56b75635e","job":"test-ns-jsqs6/test-trainjob-zx88l","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jsqs6"}
{"level":"Level(-2)","ts":"2026-11-03T04:00:15.30764757Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jsqs6","name":"test-trainjob-zx88l","reconcileID":"ecc55174-52c3-4303-8c3b-f775b5fcc7d7","job":"test-ns-jsqs6/test-trainjob-zx88l","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jsqs6"}
{"level":"Level(-2)","ts":"2026-11-03T04:00:16.203091161Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jsqs6","name":"test-trainjob-zx88l","reconcileID":"7f90dd77-e1c6-434c-9fe6-5bf2401859b7","job":"test-ns-jsqs6/test-trainjob-zx88l","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jsqs6"}
{"level":"Level(-2)","ts":"2026-11-03T04:00:16.267503518Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jsqs6","name":"test-trainjob-zx88l","reconcileID":"6477b31d-52a3-4490-a840-d4651e56ca6a","job":"test-ns-jsqs6/test-trainjob-zx88l","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jsqs6"}
{"level":"Level(-2)","ts":"2026-11-03T04:00:16.314247427Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jsqs6","name":"test-trainjob-zx88l","reconcileID":"7a2f465c-0b5b-4d0e-85fb-dc2eabadb41a","job":"test-ns-jsqs6/test-trainjob-zx88l","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jsqs6"}
{"level":"Level(-2)","ts":"2026-11-03T04:00:16.427686067Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jsqs6","name":"test-trainjob-zx88l","reconcileID":"37963b9e-ce0c-41ef-ac92-041cbc25906c","job":"test-ns-jsqs6/test-trainjob-zx88l","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jsqs6"}
{"level":"Level(-2)","ts":"2026-11-03T04:00:17.335957133Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-snzcm","name":"test-trainjob-fz7bq","reconcileID":"ade6f5a3-3298-4f39-b1cf-0029fe860b30","job":"test-ns-snzcm/test-trainjob-fz7bq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-snzcm"}
{"level":"Level(-2)","ts":"2026-11-03T04:00:17.803558418Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-snzcm","name":"test-trainjob-fz7bq","reconcileID":"8f0fc073-b9cd-4c9e-b8c0-37acfb853946","job":"test-ns-snzcm/test-trainjob-fz7bq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-snzcm"}
{"level":"Level(-2)","ts":"2026-11-03T04:00:18.031359943Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-snzcm","name":"test-trainjob-fz7bq","reconcileID":"b50b4fe7-fb6b-4a29-a2c1-3b6c4f730a51","job":"test-ns-snzcm/test-trainjob-fz7bq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-snzcm"}
{"level":"Level(-2)","ts":"2026-11-03T04:00:18.142030353Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-snzcm","name":"test-trainjob-fz7bq","reconcileID":"ecd61b19-4214-428a-9115-63d716794995","job":"test-ns-snzcm/test-trainjob-fz7bq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-snzcm"}
{"level":"Level(-2)","ts":"2026-11-03T04:00:18.356717153Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-snzcm","name":"test-trainjob-fz7bq","reconcileID":"1d8dde2b-bace-4481-996c-3671bcb8bd41","job":"test-ns-snzcm/test-trainjob-fz7bq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-snzcm"}
{"level":"Level(-2)","ts":"2026-11-03T04:00:18.46928378Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-snzcm","name":"test-trainjob-fz7bq","reconcileID":"bced0a30-62b6-4063-8762-aadffcbf4fd6","job":"test-ns-snzcm/test-trainjob-fz7bq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-snzcm"}
{"level":"Level(-2)","ts":"2026-11-03T04:01:54.788513924Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-snzcm","name":"test-trainjob-fz7bq","reconcileID":"b4626962-9614-4ad5-92ee-81c9e91ccc66","job":"test-ns-snzcm/test-trainjob-fz7bq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-snzcm"}
{"level":"Level(-2)","ts":"2026-11-03T04:01:54.899600365Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-snzcm","name":"test-trainjob-fz7bq","reconcileID":"6848d3d2-783d-4c17-8f16-15b071b8650c","job":"test-ns-snzcm/test-trainjob-fz7bq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-snzcm"}
{"level":"Level(-2)","ts":"2026-11-03T04:01:57.78840457Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-snzcm","name":"test-trainjob-fz7bq","reconcileID":"ab56216c-95ec-4df8-b6e7-96ff14c3c76f","job":"test-ns-snzcm/test-trainjob-fz7bq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-snzcm"}
{"level":"Level(-2)","ts":"2026-11-03T04:01:57.902949294Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-snzcm","name":"test-trainjob-fz7bq","reconcileID":"68e6055f-1f7c-4fa7-b405-c59e1201b49b","job":"test-ns-snzcm/test-trainjob-fz7bq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-snzcm"}
{"level":"Level(-2)","ts":"2026-11-03T04:01:58.797436674Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-snzcm","name":"test-trainjob-fz7bq","reconcileID":"d42b14fd-154f-4b96-abf9-ecd2af50795d","job":"test-ns-snzcm/test-trainjob-fz7bq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-snzcm"}
{"level":"Level(-2)","ts":"2026-11-03T04:01:58.91101773Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-snzcm","name":"test-trainjob-fz7bq","reconcileID":"62e469b0-d45c-4c12-9aab-143782a5705e","job":"test-ns-snzcm/test-trainjob-fz7bq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-snzcm"}
{"level":"Level(-2)","ts":"2026-11-03T04:01:58.915999786Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-snzcm","name":"test-trainjob-fz7bq","reconcileID":"3f37026d-1b67-4915-a4f0-ccb1ff9b7a89","job":"test-ns-snzcm/test-trainjob-fz7bq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-snzcm"}
{"level":"Level(-2)","ts":"2026-11-03T04:01:59.024093631Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-snzcm","name":"test-trainjob-fz7bq","reconcileID":"1ce823d7-44f2-46da-8da2-62e666f6b73c","job":"test-ns-snzcm/test-trainjob-fz7bq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-snzcm"}
{"level":"Level(-2)","ts":"2026-11-03T04:02:00.965271934Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-8rmqm","name":"test-trainjob-2jjmb","reconcileID":"04334887-d7a1-42d5-8b99-46634b777ad1","job":"test-ns-8rmqm/test-trainjob-2jjmb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-8rmqm"}
{"level":"Level(-2)","ts":"2026-11-03T04:02:01.536943679Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-8rmqm","name":"test-trainjob-2jjmb","reconcileID":"88cac7bd-ba71-4941-a58a-d40d4c1d6867","job":"test-ns-8rmqm/test-trainjob-2jjmb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-8rmqm"}
{"level":"Level(-2)","ts":"2026-11-03T04:02:01.721140515Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-8rmqm","name":"test-trainjob-2jjmb","reconcileID":"a192e7cb-7689-4027-9ab6-e39c518ba621","job":"test-ns-8rmqm/test-trainjob-2jjmb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-8rmqm"}
{"level":"Level(-2)","ts":"2026-11-03T04:02:01.836284111Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-8rmqm","name":"test-trainjob-2jjmb","reconcileID":"82304a70-46d5-480e-ad18-8bed223846a5","job":"test-ns-8rmqm/test-trainjob-2jjmb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-8rmqm"}
{"level":"Level(-2)","ts":"2026-11-03T04:02:02.096199979Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-8rmqm","name":"test-trainjob-2jjmb","reconcileID":"b1917e94-009a-4755-af83-913b30df29d7","job":"test-ns-8rmqm/test-trainjob-2jjmb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-8rmqm"}
{"level":"Level(-2)","ts":"2026-11-03T04:02:02.214018931Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-8rmqm","name":"test-trainjob-2jjmb","reconcileID":"d35d441d-d7dd-4902-9240-c85ca2b509a4","job":"test-ns-8rmqm/test-trainjob-2jjmb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-8rmqm"}
{"level":"Level(-2)","ts":"2026-11-03T04:02:05.322391155Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-snzcm","name":"test-trainjob-fz7bq","reconcileID":"a24e5de7-ecdf-4eed-bab4-66af560d93f5","job":"test-ns-snzcm/test-trainjob-fz7bq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-snzcm"}
{"level":"Level(-2)","ts":"2026-11-03T04:02:05.435981317Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-snzcm","name":"test-trainjob-fz7bq","reconcileID":"b6c7fc47-8ef2-434d-9f2b-e67f7b009ef9","job":"test-ns-snzcm/test-trainjob-fz7bq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-snzcm"}
{"level":"Level(-2)","ts":"2026-11-03T04:02:05.651386301Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-sw8lc","name":"test-trainjob-xlwwg","reconcileID":"26babb06-7d83-4883-95ec-4ce5770dc8ea","job":"test-ns-sw8lc/test-trainjob-xlwwg","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-sw8lc"}
{"level":"Level(-2)","ts":"2026-11-03T04:02:06.160825131Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jsqs6","name":"test-trainjob-zx88l","reconcileID":"4bb12f05-3551-411b-8d05-0c5bdc01875d","job":"test-ns-jsqs6/test-trainjob-zx88l","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jsqs6"}
{"level":"Level(-2)","ts":"2026-11-03T04:02:18.911428617Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-8rmqm","name":"test-trainjob-2jjmb","reconcileID":"3a75f46a-0c45-4856-9108-d978b27ad72d","job":"test-ns-8rmqm/test-trainjob-2jjmb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-8rmqm"}
{"level":"Level(-2)","ts":"2026-11-03T04:02:19.027728044Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-8rmqm","name":"test-trainjob-2jjmb","reconcileID":"4cc129eb-402d-4337-815b-693aa3a98573","job":"test-ns-8rmqm/test-trainjob-2jjmb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-8rmqm"}
{"level":"Level(-2)","ts":"2026-11-03T04:02:23.938588653Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-8rmqm","name":"test-trainjob-2jjmb","reconcileID":"2b4dfc30-1a89-45f9-8135-ba439fb1b832","job":"test-ns-8rmqm/test-trainjob-2jjmb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-8rmqm"}
{"level":"Level(-2)","ts":"2026-11-03T04:02:24.0548084Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-8rmqm","name":"test-trainjob-2jjmb","reconcileID":"5038d708-eb17-4561-b426-07fd691f4a6d","job":"test-ns-8rmqm/test-trainjob-2jjmb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-8rmqm"}
{"level":"Level(-2)","ts":"2026-11-03T04:02:24.947703812Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-8rmqm","name":"test-trainjob-2jjmb","reconcileID":"5ba23d22-d682-4e27-9dce-1ad0f0880f02","job":"test-ns-8rmqm/test-trainjob-2jjmb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-8rmqm"}
{"level":"Level(-2)","ts":"2026-11-03T04:02:25.063159297Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-8rmqm","name":"test-trainjob-2jjmb","reconcileID":"6bae6f40-8c79-40d6-bbf1-0a1b7e536ced","job":"test-ns-8rmqm/test-trainjob-2jjmb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-8rmqm"}
{"level":"Level(-2)","ts":"2026-11-03T04:02:25.178929767Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-8rmqm","name":"test-trainjob-2jjmb","reconcileID":"20106361-807e-4968-8f96-a998630952f7","job":"test-ns-8rmqm/test-trainjob-2jjmb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-8rmqm"}
{"level":"Level(-2)","ts":"2026-11-03T04:02:25.294462603Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-8rmqm","name":"test-trainjob-2jjmb","reconcileID":"fd3bd3d9-678e-4032-b92d-7f0c2e26ebc7","job":"test-ns-8rmqm/test-trainjob-2jjmb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-8rmqm"}
{"level":"Level(-2)","ts":"2026-11-03T04:02:25.307808495Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-8rmqm","name":"test-trainjob-2jjmb","reconcileID":"6151894a-ecaa-44f5-bf14-7c6eacff24f0","job":"test-ns-8rmqm/test-trainjob-2jjmb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-8rmqm"}
{"level":"Level(-2)","ts":"2026-11-03T04:02:25.412241358Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-8rmqm","name":"test-trainjob-2jjmb","reconcileID":"d4dde40a-4453-4e31-a385-7814b05db9d1","job":"test-ns-8rmqm/test-trainjob-2jjmb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-8rmqm"}
{"level":"Level(-2)","ts":"2026-11-03T04:04:04.444413521Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-8rmqm","name":"test-trainjob-2jjmb","reconcileID":"08ac2e15-72e4-40bd-aa8d-cbcb1c0d81bf","job":"test-ns-8rmqm/test-trainjob-2jjmb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-8rmqm"}
{"level":"Level(-2)","ts":"2026-11-03T04:04:04.565545233Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-8rmqm","name":"test-trainjob-2jjmb","reconcileID":"dd3536bd-9719-4c2f-94d2-f82f2c03a22d","job":"test-ns-8rmqm/test-trainjob-2jjmb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-8rmqm"}
{"level":"Level(-2)","ts":"2026-11-03T04:04:15.488232969Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-8rmqm","name":"test-trainjob-2jjmb","reconcileID":"9495eecf-ee17-4e36-bbc1-b10448967a1b","job":"test-ns-8rmqm/test-trainjob-2jjmb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-8rmqm"}
{"level":"Level(-2)","ts":"2026-11-03T04:04:15.602614556Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-8rmqm","name":"test-trainjob-2jjmb","reconcileID":"35ab9b22-c1c1-4852-9dab-0928e8182ffc","job":"test-ns-8rmqm/test-trainjob-2jjmb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-8rmqm"}
{"level":"Level(-2)","ts":"2026-11-03T04:04:16.498831067Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-8rmqm","name":"test-trainjob-2jjmb","reconcileID":"3d6007a6-993d-4fbe-a280-5144b1c8f5a5","job":"test-ns-8rmqm/test-trainjob-2jjmb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-8rmqm"}
{"level":"Level(-2)","ts":"2026-11-03T04:04:16.614907959Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-8rmqm","name":"test-trainjob-2jjmb","reconcileID":"59740dad-09a5-4ec5-a72a-1c4351a3aefb","job":"test-ns-8rmqm/test-trainjob-2jjmb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-8rmqm"}
{"level":"Level(-2)","ts":"2026-11-03T04:04:16.72848739Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-8rmqm","name":"test-trainjob-2jjmb","reconcileID":"81a7493d-d48b-4e2a-a2cd-37f2a15eecca","job":"test-ns-8rmqm/test-trainjob-2jjmb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-8rmqm"}
{"level":"Level(-2)","ts":"2026-11-03T04:04:16.844167043Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-8rmqm","name":"test-trainjob-2jjmb","reconcileID":"69b25bf0-8663-434c-8fbf-f0324c7aef92","job":"test-ns-8rmqm/test-trainjob-2jjmb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-8rmqm"}
{"level":"Level(-2)","ts":"2026-11-03T04:04:17.033335332Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-8rmqm","name":"test-trainjob-2jjmb","reconcileID":"7c227502-b30e-4b75-94eb-f871e6aed80c","job":"test-ns-8rmqm/test-trainjob-2jjmb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-8rmqm"}
{"level":"Level(-2)","ts":"2026-11-03T04:04:17.153128958Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-8rmqm","name":"test-trainjob-2jjmb","reconcileID":"dedf14a7-359f-45c1-ae70-2512d479c5f9","job":"test-ns-8rmqm/test-trainjob-2jjmb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-8rmqm"}
{"level":"Level(-2)","ts":"2026-11-03T04:04:26.700679228Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-8rmqm","name":"test-trainjob-2jjmb","reconcileID":"4e513a67-f259-4a2b-ab02-0df944d4e3b5","job":"test-ns-8rmqm/test-trainjob-2jjmb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-8rmqm"}
{"level":"Level(-2)","ts":"2026-11-03T04:04:26.816801294Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-8rmqm","name":"test-trainjob-2jjmb","reconcileID":"f58233b4-1624-499e-9f02-0dd61094955b","job":"test-ns-8rmqm/test-trainjob-2jjmb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-8rmqm"}
{"level":"Level(-2)","ts":"2026-11-03T04:04:26.825503821Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-8rmqm","name":"test-trainjob-2jjmb","reconcileID":"924a7c42-6f9e-46eb-80cc-141798fdcd06","job":"test-ns-8rmqm/test-trainjob-2jjmb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-8rmqm"}
{"level":"Level(-2)","ts":"2026-11-03T04:04:26.933783005Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-8rmqm","name":"test-trainjob-2jjmb","reconcileID":"2a6afe37-5766-49af-8262-f3b32b5b42d3","job":"test-ns-8rmqm/test-trainjob-2jjmb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-8rmqm"}
{"level":"Level(-2)","ts":"2026-11-03T04:04:33.401268597Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-8rmqm","name":"test-trainjob-2jjmb","reconcileID":"0e35997e-8076-48a0-8ebf-0ce202faeb5a","job":"test-ns-8rmqm/test-trainjob-2jjmb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-8rmqm"}
{"level":"Level(-2)","ts":"2026-11-03T04:06:04.426890667Z","logger":"tas-node-reconciler","caller":"tas/node_controller.go:136","msg":"Reconcile Node","replica-role":"leader","name":"ip-10-0-137-46.us-west-2.compute.internal","reconcileID":"6381d254-f41a-4359-bce1-98c9e0dd6fd6"}
{"level":"Level(-2)","ts":"2026-11-03T04:06:19.861692291Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-hsx57","name":"ifc7933d1cd1","reconcileID":"fd5c2b4e-55e5-4747-aeb0-dd03f9a8cdc2","job":"test-ns-hsx57/ifc7933d1cd1","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-hsx57"}
{"level":"Level(-2)","ts":"2026-11-03T04:06:20.33405155Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-hsx57","name":"ifc7933d1cd1","reconcileID":"3c8d3bbb-82ba-4af7-a88a-232ce8039c6a","job":"test-ns-hsx57/ifc7933d1cd1","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-hsx57"}
{"level":"Level(-2)","ts":"2026-11-03T04:06:20.56278393Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-hsx57","name":"ifc7933d1cd1","reconcileID":"a3dfb250-9ce3-4f4a-a6a5-df3cc5d4c369","job":"test-ns-hsx57/ifc7933d1cd1","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-hsx57"}
{"level":"Level(-2)","ts":"2026-11-03T04:06:20.677791055Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-hsx57","name":"ifc7933d1cd1","reconcileID":"3eebeaab-1e66-476f-b60b-643a356d5d09","job":"test-ns-hsx57/ifc7933d1cd1","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-hsx57"}
{"level":"Level(-2)","ts":"2026-11-03T04:06:21.03069034Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-hsx57","name":"ifc7933d1cd1","reconcileID":"aa968e6e-f630-44d3-8257-ad993ebf0ef8","job":"test-ns-hsx57/ifc7933d1cd1","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-hsx57"}
{"level":"Level(-2)","ts":"2026-11-03T04:06:21.1470757Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-hsx57","name":"ifc7933d1cd1","reconcileID":"41af0f76-b623-4fd2-a7e4-0534d24b72b6","job":"test-ns-hsx57/ifc7933d1cd1","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-hsx57"}
{"level":"Level(-2)","ts":"2026-11-03T04:07:21.713469125Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-hsx57","name":"ifc7933d1cd1","reconcileID":"8fe1c0aa-a872-4245-91b4-f27176a48776","job":"test-ns-hsx57/ifc7933d1cd1","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-hsx57"}
{"level":"Level(-2)","ts":"2026-11-03T04:07:21.829743145Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-hsx57","name":"ifc7933d1cd1","reconcileID":"68a91c2c-d48b-426f-b7b4-14039c8a37dc","job":"test-ns-hsx57/ifc7933d1cd1","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-hsx57"}
{"level":"Level(-2)","ts":"2026-11-03T04:07:45.67419762Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-hsx57","name":"ifc7933d1cd1","reconcileID":"42db02ec-100f-48c6-a37d-ccd01196d69e","job":"test-ns-hsx57/ifc7933d1cd1","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-hsx57"}
{"level":"Level(-2)","ts":"2026-11-03T04:07:45.79103463Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-hsx57","name":"ifc7933d1cd1","reconcileID":"b7e19ab0-c290-4c32-9b09-5fd8f8bcd39a","job":"test-ns-hsx57/ifc7933d1cd1","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-hsx57"}
{"level":"Level(-2)","ts":"2026-11-03T04:07:46.684813963Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-hsx57","name":"ifc7933d1cd1","reconcileID":"6aa84657-a223-4bcf-9ecd-abdd9c2a996b","job":"test-ns-hsx57/ifc7933d1cd1","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-hsx57"}
{"level":"Level(-2)","ts":"2026-11-03T04:07:46.798858297Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-hsx57","name":"ifc7933d1cd1","reconcileID":"01c8c98b-76c2-4451-8285-2013bd7b8ff5","job":"test-ns-hsx57/ifc7933d1cd1","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-hsx57"}
{"level":"Level(-2)","ts":"2026-11-03T04:07:46.842451389Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-hsx57","name":"ifc7933d1cd1","reconcileID":"287b6f7c-5349-4659-8e95-5cd486c3779e","job":"test-ns-hsx57/ifc7933d1cd1","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-hsx57"}
{"level":"Level(-2)","ts":"2026-11-03T04:07:46.914424129Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-hsx57","name":"ifc7933d1cd1","reconcileID":"28c908dd-782c-44dd-9cca-e4666354fc85","job":"test-ns-hsx57/ifc7933d1cd1","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-hsx57"}
{"level":"Level(-2)","ts":"2026-11-03T04:07:52.549917004Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:141","msg":"ResourceFlavor create event","replica-role":"leader","resourceFlavor":{"name":"rf-fkql6"}}
{"level":"Level(-2)","ts":"2026-11-03T04:07:52.551125547Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-fkql6","reconcileID":"79d1045b-c3a2-4ab6-87de-e8ecda8b136a"}
{"level":"Level(-2)","ts":"2026-11-03T04:07:52.898007863Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:351","msg":"ClusterQueue create event","replica-role":"leader","clusterQueue":{"name":"cq-m2r6r"}}
{"level":"Level(-2)","ts":"2026-11-03T04:07:52.898343141Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-m2r6r","reconcileID":"00ceac6c-b1c4-4d9b-b1c3-e1b497bfc45e"}
{"level":"Level(-2)","ts":"2026-11-03T04:07:52.915846399Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:240","msg":"LocalQueue create event","replica-role":"leader","localQueue":{"name":"lq-gntg5","namespace":"test-ns-pj9cs"}}
{"level":"Level(-2)","ts":"2026-11-03T04:07:52.916050895Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-pj9cs","name":"lq-gntg5","reconcileID":"5664cabe-170c-4708-a94a-d0c003493f58"}
{"level":"Level(-2)","ts":"2026-11-03T04:07:52.957718042Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-m2r6r"}}
{"level":"Level(-2)","ts":"2026-11-03T04:07:52.957917527Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-m2r6r","reconcileID":"4dc9f4d2-147e-4c6a-8e4f-b7e58f810d3e"}
{"level":"Level(-2)","ts":"2026-11-03T04:07:52.972353691Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-pj9cs","name":"lq-gntg5","reconcileID":"e9178159-00cb-4eee-bcce-d15323221033"}
{"level":"Level(-2)","ts":"2026-11-03T04:07:52.97234861Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-gntg5","namespace":"test-ns-pj9cs"}}
{"level":"Level(-2)","ts":"2026-11-03T04:07:53.030883351Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-pj9cs","name":"lq-gntg5","reconcileID":"26399ce7-6868-4df5-bfad-70acce757a2c"}
{"level":"Level(-2)","ts":"2026-11-03T04:07:53.030881786Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-gntg5","namespace":"test-ns-pj9cs"}}
{"level":"Level(-2)","ts":"2026-11-03T04:07:53.031024998Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-pj9cs","name":"lq-gntg5","reconcileID":"4eab5f53-222e-4ce5-bc2b-208f8a0c6398"}
{"level":"Level(-2)","ts":"2026-11-03T04:07:53.78366797Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-pj9cs","name":"dw-deployment-rnsbp-d7d46fb8-777pw","reconcileID":"9a607f5e-380b-4d7b-858a-6a38ce94a89c","job":"test-ns-pj9cs/dw-deployment-rnsbp-d7d46fb8-777pw","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-03T04:07:53.836394962Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1.PriorityClass","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"debug","ts":"2026-11-03T04:07:54.288394538Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"dw-deployment-rnsbp-d7d46fb8-777pw","namespace":"test-ns-pj9cs"},"kind":"Pod","apiVersion":"v1","type":"Normal","reason":"CreatedWorkload","action":"CreatedWorkload","note":"Created Workload: test-ns-pj9cs/pod-dw-deployment-rnsbp-d7d46fb8-777pw-383b1"}
{"level":"Level(-2)","ts":"2026-11-03T04:07:54.338536095Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1276","msg":"Workload create event","replica-role":"leader","workload":{"name":"pod-dw-deployment-rnsbp-d7d46fb8-777pw-383b1","namespace":"test-ns-pj9cs"},"queue":"lq-gntg5","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-03T04:07:54.338569017Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-pj9cs","name":"pod-dw-deployment-rnsbp-d7d46fb8-777pw-383b1","reconcileID":"b4d02a78-b9e1-4f37-9c25-95ccb3c9c44c"}
{"level":"Level(-2)","ts":"2026-11-03T04:07:54.338607778Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-pj9cs","name":"dw-deployment-rnsbp-d7d46fb8-777pw","reconcileID":"92985d16-aeac-4ced-bfc4-ec9e2f969f7e","job":"test-ns-pj9cs/dw-deployment-rnsbp-d7d46fb8-777pw","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-03T04:07:54.338633146Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-pj9cs","name":"pod-dw-deployment-rnsbp-d7d46fb8-777pw-383b1","reconcileID":"b4d02a78-b9e1-4f37-9c25-95ccb3c9c44c","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-03T04:07:54.338819613Z","logger":"scheduler","caller":"scheduler/scheduler.go:308","msg":"Obtained heads","replica-role":"follower","schedulingCycle":1,"headCount":1,"waitDuration":2116.702454291}
{"level":"Level(-2)","ts":"2026-11-03T04:07:54.338852683Z","logger":"scheduler","caller":"scheduler/scheduler.go:323","msg":"Snapshot taken","replica-role":"follower","schedulingCycle":1,"duration":0.000011554}
{"level":"Level(-2)","ts":"2026-11-03T04:07:54.338865352Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-pj9cs","name":"pod-dw-deployment-rnsbp-d7d46fb8-777pw-383b1","reconcileID":"e68d1913-eefa-4540-98b7-2df50c29a6b2"}
{"level":"Level(-2)","ts":"2026-11-03T04:07:54.340098763Z","logger":"scheduler","caller":"scheduler/scheduler.go:328","msg":"Nomination done","replica-role":"follower","schedulingCycle":1,"entries":1,"inadmissibleEntries":0,"duration":0.001239001}
{"level":"Level(-2)","ts":"2026-11-03T04:07:54.340122568Z","logger":"scheduler","caller":"scheduler/scheduler.go:406","msg":"Attempting to schedule workload","replica-role":"follower","schedulingCycle":1,"workload":{"name":"pod-dw-deployment-rnsbp-d7d46fb8-777pw-383b1","namespace":"test-ns-pj9cs"},"clusterQueue":{"name":"cq-m2r6r"}}
{"level":"Level(-2)","ts":"2026-11-03T04:07:54.340215861Z","logger":"scheduler","caller":"scheduler/scheduler.go:1041","msg":"Workload assumed in the cache","replica-role":"follower","schedulingCycle":1,"workload":{"name":"pod-dw-deployment-rnsbp-d7d46fb8-777pw-383b1","namespace":"test-ns-pj9cs"},"clusterQueue":{"name":"cq-m2r6r"}}
{"level":"Level(-2)","ts":"2026-11-03T04:07:54.34255448Z","logger":"scheduler","caller":"scheduler/scheduler.go:362","msg":"Workload processing done","replica-role":"follower","schedulingCycle":1,"duration":0.002435594}
{"level":"Level(-2)","ts":"2026-11-03T04:07:54.342582255Z","logger":"scheduler","caller":"scheduler/scheduler.go:297","msg":"Scheduling cycle complete","replica-role":"follower","schedulingCycle":1,"duration":2116.706217303}
{"level":"Level(-2)","ts":"2026-11-03T04:07:54.342599895Z","logger":"scheduler","caller":"scheduler/scheduler.go:295","msg":"Scheduling cycle starts","replica-role":"leader","schedulingCycle":2}
{"level":"Level(-2)","ts":"2026-11-03T04:07:54.460844919Z","logger":"scheduler","caller":"scheduler/scheduler.go:1000","msg":"Workload successfully admitted and assigned flavors","replica-role":"follower","schedulingCycle":1,"workload":{"name":"pod-dw-deployment-rnsbp-d7d46fb8-777pw-383b1","namespace":"test-ns-pj9cs"},"clusterQueue":{"name":"cq-m2r6r"},"assignments":[{"name":"main","flavors":{"cpu":"rf-fkql6","memory":"rf-fkql6"},"resourceUsage":{"cpu":"1","memory":"3Gi"},"count":1}]}
{"level":"debug","ts":"2026-11-03T04:07:54.460885144Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"pod-dw-deployment-rnsbp-d7d46fb8-777pw-383b1","namespace":"test-ns-pj9cs"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"QuotaReserved","action":"QuotaReserved","note":"Quota reserved in ClusterQueue cq-m2r6r, wait time since queued was 0s"}
{"level":"debug","ts":"2026-11-03T04:07:54.460921112Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"pod-dw-deployment-rnsbp-d7d46fb8-777pw-383b1","namespace":"test-ns-pj9cs"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"Admitted","action":"Admitted","note":"Admitted by ClusterQueue cq-m2r6r, wait time since reservation was 0s"}
{"level":"Level(-2)","ts":"2026-11-03T04:07:54.46091375Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-pj9cs","name":"pod-dw-deployment-rnsbp-d7d46fb8-777pw-383b1","reconcileID":"3f282e59-f777-45eb-a85e-8b5950d2059f"}
{"level":"Level(-2)","ts":"2026-11-03T04:07:54.460936618Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"pod-dw-deployment-rnsbp-d7d46fb8-777pw-383b1","namespace":"test-ns-pj9cs"},"queue":"lq-gntg5","status":"admitted","prevStatus":"pending","clusterQueue":"cq-m2r6r"}
{"level":"Level(-2)","ts":"2026-11-03T04:07:54.461024795Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-pj9cs","name":"pod-dw-deployment-rnsbp-d7d46fb8-777pw-383b1","reconcileID":"3f282e59-f777-45eb-a85e-8b5950d2059f","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-03T04:07:54.461031964Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-pj9cs","name":"dw-deployment-rnsbp-d7d46fb8-777pw","reconcileID":"6930ef44-faea-424a-b3f1-8f0846e9d44d","job":"test-ns-pj9cs/dw-deployment-rnsbp-d7d46fb8-777pw","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-03T04:07:54.461240461Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-pj9cs","name":"pod-dw-deployment-rnsbp-d7d46fb8-777pw-383b1","reconcileID":"d12fc8db-f7ee-4974-aff1-b1cd9295bdda"}
{"level":"Level(-2)","ts":"2026-11-03T04:07:54.461382834Z","logger":"v1_pod","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-pj9cs","name":"dw-deployment-rnsbp-d7d46fb8-777pw","reconcileID":"6930ef44-faea-424a-b3f1-8f0846e9d44d","job":"test-ns-pj9cs/dw-deployment-rnsbp-d7d46fb8-777pw","gvk":"/v1, Kind=Pod"}
{"level":"debug","ts":"2026-11-03T04:07:54.582150698Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"dw-deployment-rnsbp-d7d46fb8-777pw","namespace":"test-ns-pj9cs"},"kind":"Pod","apiVersion":"v1","type":"Normal","reason":"Started","action":"Started","note":"Admitted by clusterQueue cq-m2r6r"}
{"level":"Level(-2)","ts":"2026-11-03T04:07:54.582335404Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-pj9cs","name":"dw-deployment-rnsbp-d7d46fb8-777pw","reconcileID":"a3145c9d-d88d-4d98-9703-fafcc30c84a2","job":"test-ns-pj9cs/dw-deployment-rnsbp-d7d46fb8-777pw","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-03T04:07:54.589750365Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-pj9cs","name":"dw-deployment-rnsbp-d7d46fb8-777pw","reconcileID":"66fba5e2-1c24-48da-9629-55cf15df2de1","job":"test-ns-pj9cs/dw-deployment-rnsbp-d7d46fb8-777pw","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-03T04:07:54.650131498Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-pj9cs","name":"dw-deployment-rnsbp-d7d46fb8-777pw","reconcileID":"83abd810-ac13-42ad-a8b3-6674abf55e6a","job":"test-ns-pj9cs/dw-deployment-rnsbp-d7d46fb8-777pw","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-03T04:07:54.755253419Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-pj9cs","name":"dw-deployment-rnsbp-d7d46fb8-777pw","reconcileID":"875e7b1b-97c1-401a-bfc4-8b494438e1bf","job":"test-ns-pj9cs/dw-deployment-rnsbp-d7d46fb8-777pw","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-03T04:07:55.339430638Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-pj9cs","name":"lq-gntg5","reconcileID":"80371524-e5a2-43c6-a8a1-bd015e4512db"}
{"level":"Level(-2)","ts":"2026-11-03T04:07:55.339441626Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-m2r6r","reconcileID":"c85b45f6-6ce1-4609-b2e9-a421bbdf2961"}
{"level":"Level(-2)","ts":"2026-11-03T04:07:55.398167575Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-gntg5","namespace":"test-ns-pj9cs"}}
{"level":"Level(-2)","ts":"2026-11-03T04:07:55.3982912Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-pj9cs","name":"lq-gntg5","reconcileID":"4a2bc3b4-d671-451f-8b88-da8c671501a0"}
{"level":"Level(-2)","ts":"2026-11-03T04:07:55.40196212Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-m2r6r"}}
{"level":"Level(-2)","ts":"2026-11-03T04:07:55.402137583Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-m2r6r","reconcileID":"71c429cf-2470-46bf-9e67-97c2002e878a"}
{"level":"Level(-2)","ts":"2026-11-03T04:07:55.582750294Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-pj9cs","name":"pod-dw-deployment-rnsbp-d7d46fb8-777pw-383b1","reconcileID":"8c9369df-cb53-4e92-9670-fb425212b39f"}
{"level":"Level(-2)","ts":"2026-11-03T04:07:55.607007986Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-pj9cs","name":"dw-deployment-rnsbp-d7d46fb8-777pw","reconcileID":"312c9297-a5c6-430a-8818-d68c85609972","job":"test-ns-pj9cs/dw-deployment-rnsbp-d7d46fb8-777pw","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-03T04:07:56.607358316Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-pj9cs","name":"pod-dw-deployment-rnsbp-d7d46fb8-777pw-383b1","reconcileID":"8edc25bd-45d6-4736-8300-897e3c0087a2"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:03.968432222Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-pj9cs","name":"dw-deployment-rnsbp-d7d46fb8-777pw","reconcileID":"01e876bc-e49f-4a89-8cf9-f37f7ab8b626","job":"test-ns-pj9cs/dw-deployment-rnsbp-d7d46fb8-777pw","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:04.96867813Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-pj9cs","name":"pod-dw-deployment-rnsbp-d7d46fb8-777pw-383b1","reconcileID":"4e81306c-87b0-4298-8209-3edc9e41affd"}
{"level":"info","ts":"2026-11-03T04:09:15.990655513Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:117","msg":"Validating create","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"v21bdac94b06","namespace":"test-ns-pj9cs"},"namespace":"test-ns-pj9cs","name":"v21bdac94b06","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:redhat-ods-applications:trainer-e2e","requestID":"a9bf7034-dbd8-4469-8b77-3bb6a5eef7dd","replica-role":"leader"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:16.058930916Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-pj9cs","name":"v21bdac94b06","reconcileID":"2257777d-7452-4976-bcf5-a75f8599bed3","job":"test-ns-pj9cs/v21bdac94b06","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"debug","ts":"2026-11-03T04:09:16.386006525Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"v21bdac94b06","namespace":"test-ns-pj9cs"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"CreatedWorkload","action":"CreatedWorkload","note":"Created Workload: test-ns-pj9cs/trainjob-v21bdac94b06-a0214"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:16.386258181Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1276","msg":"Workload create event","replica-role":"leader","workload":{"name":"trainjob-v21bdac94b06-a0214","namespace":"test-ns-pj9cs"},"queue":"lq-gntg5","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:16.386266202Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-pj9cs","name":"trainjob-v21bdac94b06-a0214","reconcileID":"ca25c25f-8701-49cb-a95d-55b21cb31ce5"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:16.386317556Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-pj9cs","name":"trainjob-v21bdac94b06-a0214","reconcileID":"ca25c25f-8701-49cb-a95d-55b21cb31ce5","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-03T04:09:16.38640107Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-pj9cs","name":"v21bdac94b06","reconcileID":"2ce67b64-58c7-4147-b773-2c841c60305f","job":"test-ns-pj9cs/v21bdac94b06","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:16.386501645Z","logger":"scheduler","caller":"scheduler/scheduler.go:308","msg":"Obtained heads","replica-role":"leader","schedulingCycle":2,"headCount":1,"waitDuration":82.043901328}
{"level":"Level(-2)","ts":"2026-11-03T04:09:16.386529024Z","logger":"scheduler","caller":"scheduler/scheduler.go:323","msg":"Snapshot taken","replica-role":"leader","schedulingCycle":2,"duration":0.000010598}
{"level":"Level(-2)","ts":"2026-11-03T04:09:16.386537088Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-pj9cs","name":"trainjob-v21bdac94b06-a0214","reconcileID":"9c0e5614-ad73-4cf2-81f3-2a66024092d9"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:16.386615788Z","logger":"scheduler","caller":"scheduler/scheduler.go:328","msg":"Nomination done","replica-role":"leader","schedulingCycle":2,"entries":1,"inadmissibleEntries":0,"duration":0.000079754}
{"level":"Level(-2)","ts":"2026-11-03T04:09:16.38663873Z","logger":"scheduler","caller":"scheduler/scheduler.go:406","msg":"Attempting to schedule workload","replica-role":"leader","schedulingCycle":2,"workload":{"name":"trainjob-v21bdac94b06-a0214","namespace":"test-ns-pj9cs"},"clusterQueue":{"name":"cq-m2r6r"}}
{"level":"Level(-2)","ts":"2026-11-03T04:09:16.386718717Z","logger":"scheduler","caller":"scheduler/scheduler.go:1041","msg":"Workload assumed in the cache","replica-role":"leader","schedulingCycle":2,"workload":{"name":"trainjob-v21bdac94b06-a0214","namespace":"test-ns-pj9cs"},"clusterQueue":{"name":"cq-m2r6r"}}
{"level":"Level(-2)","ts":"2026-11-03T04:09:16.386740631Z","logger":"scheduler","caller":"scheduler/scheduler.go:362","msg":"Workload processing done","replica-role":"leader","schedulingCycle":2,"duration":0.000105702}
{"level":"Level(-2)","ts":"2026-11-03T04:09:16.386752908Z","logger":"scheduler","caller":"scheduler/scheduler.go:297","msg":"Scheduling cycle complete","replica-role":"leader","schedulingCycle":2,"duration":82.044152885}
{"level":"Level(-2)","ts":"2026-11-03T04:09:16.386765931Z","logger":"scheduler","caller":"scheduler/scheduler.go:295","msg":"Scheduling cycle starts","replica-role":"leader","schedulingCycle":3}
{"level":"Level(-2)","ts":"2026-11-03T04:09:16.507644624Z","logger":"scheduler","caller":"scheduler/scheduler.go:1000","msg":"Workload successfully admitted and assigned flavors","replica-role":"leader","schedulingCycle":2,"workload":{"name":"trainjob-v21bdac94b06-a0214","namespace":"test-ns-pj9cs"},"clusterQueue":{"name":"cq-m2r6r"},"assignments":[{"name":"node","flavors":{"cpu":"rf-fkql6","memory":"rf-fkql6"},"resourceUsage":{"cpu":"4","memory":"16Gi"},"count":2}]}
{"level":"debug","ts":"2026-11-03T04:09:16.507699593Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-v21bdac94b06-a0214","namespace":"test-ns-pj9cs"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"Admitted","action":"Admitted","note":"Admitted by ClusterQueue cq-m2r6r, wait time since reservation was 1s"}
{"level":"debug","ts":"2026-11-03T04:09:16.507714704Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-v21bdac94b06-a0214","namespace":"test-ns-pj9cs"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"QuotaReserved","action":"QuotaReserved","note":"Quota reserved in ClusterQueue cq-m2r6r, wait time since queued was 1s"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:16.508319738Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-v21bdac94b06-a0214","namespace":"test-ns-pj9cs"},"queue":"lq-gntg5","status":"admitted","prevStatus":"pending","clusterQueue":"cq-m2r6r"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:16.508345251Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-pj9cs","name":"trainjob-v21bdac94b06-a0214","reconcileID":"2ac548e5-aebf-4479-8881-2da0953c75c6"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:16.508400733Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-pj9cs","name":"trainjob-v21bdac94b06-a0214","reconcileID":"2ac548e5-aebf-4479-8881-2da0953c75c6","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-03T04:09:16.508539775Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-pj9cs","name":"trainjob-v21bdac94b06-a0214","reconcileID":"5c013ba1-eeaf-467a-bbb5-0b48f299930d"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:16.554615435Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-pj9cs","name":"v21bdac94b06","reconcileID":"b8eed390-ae35-4c5c-96af-c13977a327ad","job":"test-ns-pj9cs/v21bdac94b06","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:16.716219925Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-pj9cs","name":"v21bdac94b06","reconcileID":"b8eed390-ae35-4c5c-96af-c13977a327ad","job":"test-ns-pj9cs/v21bdac94b06","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-3)","ts":"2026-11-03T04:09:16.830093071Z","logger":"trainjob","caller":"jobframework/reconciler.go:594","msg":"Unsuspending job","replica-role":"leader","namespace":"test-ns-pj9cs","name":"v21bdac94b06","reconcileID":"b8eed390-ae35-4c5c-96af-c13977a327ad","job":"test-ns-pj9cs/v21bdac94b06","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"v21bdac94b06\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-3)","ts":"2026-11-03T04:09:16.830179024Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-pj9cs","name":"v21bdac94b06","reconcileID":"b8eed390-ae35-4c5c-96af-c13977a327ad","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"v21bdac94b06\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:16.830303317Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-pj9cs","name":"v21bdac94b06","reconcileID":"faa4138f-559b-4096-a426-3f09cc17c5ea","job":"test-ns-pj9cs/v21bdac94b06","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:16.991718162Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-pj9cs","name":"v21bdac94b06","reconcileID":"faa4138f-559b-4096-a426-3f09cc17c5ea","job":"test-ns-pj9cs/v21bdac94b06","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-3)","ts":"2026-11-03T04:09:17.105622628Z","logger":"trainjob","caller":"jobframework/reconciler.go:594","msg":"Unsuspending job","replica-role":"leader","namespace":"test-ns-pj9cs","name":"v21bdac94b06","reconcileID":"faa4138f-559b-4096-a426-3f09cc17c5ea","job":"test-ns-pj9cs/v21bdac94b06","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"v21bdac94b06\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-3)","ts":"2026-11-03T04:09:17.105696693Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-pj9cs","name":"v21bdac94b06","reconcileID":"faa4138f-559b-4096-a426-3f09cc17c5ea","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"v21bdac94b06\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:17.105802677Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-pj9cs","name":"v21bdac94b06","reconcileID":"27cf1db9-c525-4279-aab8-4f2fd4f2ffb4","job":"test-ns-pj9cs/v21bdac94b06","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:17.267594379Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-pj9cs","name":"v21bdac94b06","reconcileID":"27cf1db9-c525-4279-aab8-4f2fd4f2ffb4","job":"test-ns-pj9cs/v21bdac94b06","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"info","ts":"2026-11-03T04:09:17.384108267Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"v21bdac94b06","namespace":"test-ns-pj9cs"},"namespace":"test-ns-pj9cs","name":"v21bdac94b06","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"11b46bdb-5aff-4b81-8395-d99640aefb73","replica-role":"leader"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:17.386928202Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-m2r6r","reconcileID":"7064d56c-7e36-4c2c-8033-2167124de566"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:17.386983905Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-pj9cs","name":"lq-gntg5","reconcileID":"7a96bd52-db23-43c8-9ec4-3dad995f74ea"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:17.443131691Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-gntg5","namespace":"test-ns-pj9cs"}}
{"level":"Level(-2)","ts":"2026-11-03T04:09:17.44329083Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-pj9cs","name":"lq-gntg5","reconcileID":"7f097cf4-56b0-44bc-b6f1-5b0fae4fba22"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:17.446462147Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-m2r6r"}}
{"level":"Level(-2)","ts":"2026-11-03T04:09:17.446650499Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-m2r6r","reconcileID":"bcf10e95-f74e-4ae1-b291-ebd195ebf534"}
{"level":"info","ts":"2026-11-03T04:09:17.557436638Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"v21bdac94b06","namespace":"test-ns-pj9cs"},"namespace":"test-ns-pj9cs","name":"v21bdac94b06","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"e9162bfc-a964-4f29-ba5d-6d5cd69c1424","replica-role":"leader"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:17.615415972Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-pj9cs","name":"v21bdac94b06","reconcileID":"22da39ed-c055-4418-8647-244a87f1af50","job":"test-ns-pj9cs/v21bdac94b06","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"debug","ts":"2026-11-03T04:09:17.615421927Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"v21bdac94b06","namespace":"test-ns-pj9cs"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"Started","action":"Started","note":"Admitted by clusterQueue cq-m2r6r"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:17.778840323Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-pj9cs","name":"v21bdac94b06","reconcileID":"22da39ed-c055-4418-8647-244a87f1af50","job":"test-ns-pj9cs/v21bdac94b06","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-3)","ts":"2026-11-03T04:09:17.889695336Z","logger":"trainjob","caller":"jobframework/reconciler.go:594","msg":"Unsuspending job","replica-role":"leader","namespace":"test-ns-pj9cs","name":"v21bdac94b06","reconcileID":"22da39ed-c055-4418-8647-244a87f1af50","job":"test-ns-pj9cs/v21bdac94b06","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"v21bdac94b06\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-3)","ts":"2026-11-03T04:09:17.889774139Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-pj9cs","name":"v21bdac94b06","reconcileID":"22da39ed-c055-4418-8647-244a87f1af50","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"v21bdac94b06\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:17.889900703Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-pj9cs","name":"v21bdac94b06","reconcileID":"787f3eb4-6da1-4c68-8440-328e8ad5badd","job":"test-ns-pj9cs/v21bdac94b06","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:18.052608737Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-pj9cs","name":"v21bdac94b06","reconcileID":"36fde628-f28b-455c-a755-5268a6eb7f0d","job":"test-ns-pj9cs/v21bdac94b06","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:18.215120412Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-pj9cs","name":"v21bdac94b06","reconcileID":"f80511e4-cec7-4026-a8db-dbf2cdf00010","job":"test-ns-pj9cs/v21bdac94b06","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:18.238867825Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1.PartialObjectMetadata","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:18.339659279Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1.PartialObjectMetadata","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:18.377133414Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-pj9cs","name":"v21bdac94b06","reconcileID":"f809c2b2-1b21-4fe5-90ef-381d6bf48283","job":"test-ns-pj9cs/v21bdac94b06","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:18.857748742Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-pj9cs","name":"v21bdac94b06","reconcileID":"109ee425-5882-4fd5-b544-061fb0a2f9b2","job":"test-ns-pj9cs/v21bdac94b06","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:19.020409573Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-pj9cs","name":"v21bdac94b06","reconcileID":"1fea0d24-abda-4db6-8eac-164f7868dce1","job":"test-ns-pj9cs/v21bdac94b06","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:19.608575943Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-pj9cs","name":"trainjob-v21bdac94b06-a0214","reconcileID":"846b6363-89df-4d10-af5e-b62f6e17a297"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:20.69172225Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-pj9cs","name":"trainjob-v21bdac94b06-a0214","reconcileID":"e138895b-ec68-406f-908f-e4593188c001"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:21.788755531Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-pj9cs","name":"v21bdac94b06","reconcileID":"325c0fd1-6205-46cc-a056-e2aa2d9b4109","job":"test-ns-pj9cs/v21bdac94b06","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:21.95307987Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-pj9cs","name":"v21bdac94b06","reconcileID":"18eb0ebb-0589-43a9-a57f-3e66778e081f","job":"test-ns-pj9cs/v21bdac94b06","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:22.091685462Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-pj9cs","name":"trainjob-v21bdac94b06-a0214","reconcileID":"60f2d06b-ed14-4cce-b4f5-a121c6de3bf6"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:46.264009872Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-pj9cs","name":"trainjob-v21bdac94b06-a0214","reconcileID":"5657e6ff-fea5-402e-88c2-1269e9fe2134"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:46.288127567Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-pj9cs","name":"v21bdac94b06","reconcileID":"cf8542d1-1988-4f11-8142-84dcedb1d3c6","job":"test-ns-pj9cs/v21bdac94b06","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:46.450206365Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-pj9cs","name":"v21bdac94b06","reconcileID":"0b45f2f0-933b-47ac-acf2-4bc21bcfcf8b","job":"test-ns-pj9cs/v21bdac94b06","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:47.299660097Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-pj9cs","name":"v21bdac94b06","reconcileID":"bbd137a9-741a-4041-b9c7-031884ff2bf9","job":"test-ns-pj9cs/v21bdac94b06","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:47.365910973Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-pj9cs","name":"trainjob-v21bdac94b06-a0214","reconcileID":"851edfa9-5881-419c-b398-edcef2b57f82"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:47.46459462Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-pj9cs","name":"v21bdac94b06","reconcileID":"94679a81-7703-4957-a65a-927432936cae","job":"test-ns-pj9cs/v21bdac94b06","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:47.626775248Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-pj9cs","name":"v21bdac94b06","reconcileID":"d733e237-2920-4020-a18e-86eab90f6681","job":"test-ns-pj9cs/v21bdac94b06","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"debug","ts":"2026-11-03T04:09:47.905111546Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"v21bdac94b06","namespace":"test-ns-pj9cs"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"FinishedWorkload","action":"FinishedWorkload","note":"Workload 'test-ns-pj9cs/trainjob-v21bdac94b06-a0214' is declared finished"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:47.905456823Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-v21bdac94b06-a0214","namespace":"test-ns-pj9cs"},"queue":"lq-gntg5","status":"finished","prevStatus":"admitted","clusterQueue":"cq-m2r6r"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:47.905515395Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-pj9cs","name":"trainjob-v21bdac94b06-a0214","reconcileID":"992ca08f-f789-43e7-a087-da76c506408a"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:47.905574532Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-pj9cs","name":"trainjob-v21bdac94b06-a0214","reconcileID":"992ca08f-f789-43e7-a087-da76c506408a","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-03T04:09:47.905632604Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-pj9cs","name":"v21bdac94b06","reconcileID":"e2a09290-59fc-45d5-ac2e-76eadf6d587c","job":"test-ns-pj9cs/v21bdac94b06","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:47.90568931Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-pj9cs","name":"trainjob-v21bdac94b06-a0214","reconcileID":"b5a275b2-5821-4574-9f5b-d4ab857642e7"}
{"level":"debug","ts":"2026-11-03T04:09:48.067525085Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"v21bdac94b06","namespace":"test-ns-pj9cs"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"FinishedWorkload","action":"FinishedWorkload","note":"Workload 'test-ns-pj9cs/trainjob-v21bdac94b06-a0214' is declared finished"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:48.184307389Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-v21bdac94b06-a0214","namespace":"test-ns-pj9cs"},"queue":"lq-gntg5","status":"finished","clusterQueue":"cq-m2r6r"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:48.184336382Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-pj9cs","name":"trainjob-v21bdac94b06-a0214","reconcileID":"c43aa00b-88af-440f-8f6a-2b2531a1adde"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:48.184471547Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-pj9cs","name":"trainjob-v21bdac94b06-a0214","reconcileID":"c43aa00b-88af-440f-8f6a-2b2531a1adde","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-03T04:09:48.184512397Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-pj9cs","name":"trainjob-v21bdac94b06-a0214","reconcileID":"6cd4ab82-3ec9-47af-8446-a1aa04f799ed"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:48.185110445Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-pj9cs","name":"v21bdac94b06","reconcileID":"0554e9c0-3fd7-49ce-a390-249b533f7ff5","job":"test-ns-pj9cs/v21bdac94b06","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"debug","ts":"2026-11-03T04:09:48.347300184Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"v21bdac94b06","namespace":"test-ns-pj9cs"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"FinishedWorkload","action":"FinishedWorkload","note":"Workload 'test-ns-pj9cs/trainjob-v21bdac94b06-a0214' is declared finished"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:48.906649465Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-pj9cs","name":"lq-gntg5","reconcileID":"833ba6b4-fc02-4ae6-9e0f-46538d827fcc"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:48.906650148Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-m2r6r","reconcileID":"c7930998-d3a2-44bd-a1c3-89ba738264dc"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:48.963770209Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-gntg5","namespace":"test-ns-pj9cs"}}
{"level":"Level(-2)","ts":"2026-11-03T04:09:48.963908232Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-pj9cs","name":"lq-gntg5","reconcileID":"0fc3fb17-042e-42f2-b87c-706b6343cb06"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:48.966395546Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-m2r6r"}}
{"level":"Level(-2)","ts":"2026-11-03T04:09:48.966558573Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-m2r6r","reconcileID":"40452c74-f6f7-43a9-83a1-ecc6084198ab"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:49.479828901Z","logger":"trainjob","caller":"workload/workload.go:1810","msg":"Workload is orphaned; finishing to release quota","replica-role":"leader","namespace":"test-ns-pj9cs","name":"v21bdac94b06","reconcileID":"34a6e4d7-1293-4e8c-9edf-fc868768d301","job":"test-ns-pj9cs/v21bdac94b06","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:49.529857833Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1312","msg":"Workload delete event","replica-role":"leader","workload":{"name":"trainjob-v21bdac94b06-a0214","namespace":"test-ns-pj9cs"},"queue":"lq-gntg5","status":"finished"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:49.529861649Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-pj9cs","name":"trainjob-v21bdac94b06-a0214","reconcileID":"90c1fc74-1469-4a33-bd5a-25fd068e2416"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:49.529916312Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-pj9cs","name":"trainjob-v21bdac94b06-a0214","reconcileID":"90c1fc74-1469-4a33-bd5a-25fd068e2416","isDeleted":true}
{"level":"Level(-2)","ts":"2026-11-03T04:09:50.503134982Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-m2r6r"}}
{"level":"Level(-2)","ts":"2026-11-03T04:09:50.503236177Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-m2r6r","reconcileID":"19fcfa97-14b7-48d0-878e-138d135c93f0"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:50.503318927Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-pj9cs","name":"pod-dw-deployment-rnsbp-d7d46fb8-777pw-383b1","reconcileID":"76068cc8-2a14-4275-b077-9599fd5440fd"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:50.521762181Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:171","msg":"ResourceFlavor update event","replica-role":"leader","resourceFlavor":{"name":"rf-fkql6"}}
{"level":"Level(-2)","ts":"2026-11-03T04:09:50.521909081Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-fkql6","reconcileID":"f3d1be39-d38a-47b9-9d1b-bfc4b08473f9"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:50.530978179Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-pj9cs","name":"lq-gntg5","reconcileID":"5412d9ef-441e-45c8-95cc-00b37a42055b"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:50.562524351Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-m2r6r"}}
{"level":"Level(-2)","ts":"2026-11-03T04:09:50.562609816Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-m2r6r","reconcileID":"7a6d02d7-4816-407e-aca1-b7c5aa76c025"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:50.56260815Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-pj9cs","name":"lq-gntg5","reconcileID":"4d9d7bbc-aa97-45bd-b5ef-7c6cf552c7f9"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:50.562703486Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-pj9cs","name":"pod-dw-deployment-rnsbp-d7d46fb8-777pw-383b1","reconcileID":"bd6782c1-e663-4ddb-8d20-034cf93542cd"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:50.62102999Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-gntg5","namespace":"test-ns-pj9cs"}}
{"level":"Level(-2)","ts":"2026-11-03T04:09:50.621201631Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-pj9cs","name":"lq-gntg5","reconcileID":"399ea342-e5c6-4a5c-b99c-aaa02d9ff056"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:51.150676685Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:141","msg":"ResourceFlavor create event","replica-role":"leader","resourceFlavor":{"name":"rf-j9bhm"}}
{"level":"Level(-2)","ts":"2026-11-03T04:09:51.150800038Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-j9bhm","reconcileID":"a62adfcb-92a9-4f36-a3c9-8d35c942f8fc"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:51.486021814Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-pj9cs","name":"pod-dw-deployment-rnsbp-d7d46fb8-777pw-383b1","reconcileID":"953319d6-2fb4-4fb1-b7ba-5a04a3dc78b4"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:51.499500751Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:351","msg":"ClusterQueue create event","replica-role":"leader","clusterQueue":{"name":"cq-s4rfg"}}
{"level":"Level(-2)","ts":"2026-11-03T04:09:51.499687749Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-s4rfg","reconcileID":"b8f8ba3c-7647-4fa4-a10a-ae21c2f729e1"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:51.516903397Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:240","msg":"LocalQueue create event","replica-role":"leader","localQueue":{"name":"lq-7qwch","namespace":"test-ns-289zf"}}
{"level":"Level(-2)","ts":"2026-11-03T04:09:51.517079056Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-289zf","name":"lq-7qwch","reconcileID":"83a7e5dd-1ac2-4bab-b9a7-1db777627678"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:51.558438271Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-s4rfg"}}
{"level":"Level(-2)","ts":"2026-11-03T04:09:51.558610916Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-s4rfg","reconcileID":"ce7e1cf6-8bc4-42ed-9950-5a867f35993f"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:51.574793203Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-7qwch","namespace":"test-ns-289zf"}}
{"level":"Level(-2)","ts":"2026-11-03T04:09:51.574883293Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-289zf","name":"lq-7qwch","reconcileID":"84cdb1a8-c475-434f-ad26-c7177f635b7e"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:51.63139314Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-7qwch","namespace":"test-ns-289zf"}}
{"level":"Level(-2)","ts":"2026-11-03T04:09:51.631447896Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-289zf","name":"lq-7qwch","reconcileID":"1b07729d-5035-4697-829a-3ef50d6696c2"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:51.631553097Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-289zf","name":"lq-7qwch","reconcileID":"4fa1785b-0485-4c79-bf9d-d4aec10e2222"}
{"level":"info","ts":"2026-11-03T04:09:51.651002466Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:117","msg":"Validating create","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-preemption-trainjob-fnt6p","namespace":"test-ns-289zf"},"namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:admin","requestID":"4a64998f-ec66-4ca7-883d-d069ece8cac9","replica-role":"leader"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:51.707317931Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"3da1a850-3471-49a3-9a8a-1232ef05e872","job":"test-ns-289zf/test-preemption-trainjob-fnt6p","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:52.029856753Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"590865f8-7992-441d-9b24-39d70212d612","job":"test-ns-289zf/test-preemption-trainjob-fnt6p","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"debug","ts":"2026-11-03T04:09:52.02986463Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"test-preemption-trainjob-fnt6p","namespace":"test-ns-289zf"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"CreatedWorkload","action":"CreatedWorkload","note":"Created Workload: test-ns-289zf/trainjob-test-preemption-trainjob-fnt6p-dced9"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:52.029879298Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1276","msg":"Workload create event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-fnt6p-dced9","namespace":"test-ns-289zf"},"queue":"lq-7qwch","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:52.029925918Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-289zf","name":"trainjob-test-preemption-trainjob-fnt6p-dced9","reconcileID":"d460ac27-0922-4261-ae34-1cec498dab3d"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:52.029972292Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-289zf","name":"trainjob-test-preemption-trainjob-fnt6p-dced9","reconcileID":"d460ac27-0922-4261-ae34-1cec498dab3d","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-03T04:09:52.030102599Z","logger":"scheduler","caller":"scheduler/scheduler.go:308","msg":"Obtained heads","replica-role":"leader","schedulingCycle":3,"headCount":1,"waitDuration":35.643334412}
{"level":"Level(-2)","ts":"2026-11-03T04:09:52.030120816Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-289zf","name":"trainjob-test-preemption-trainjob-fnt6p-dced9","reconcileID":"225552d2-4196-48e6-815c-4a07a5b9ab46"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:52.030163803Z","logger":"scheduler","caller":"scheduler/scheduler.go:323","msg":"Snapshot taken","replica-role":"leader","schedulingCycle":3,"duration":0.000016338}
{"level":"Level(-2)","ts":"2026-11-03T04:09:52.030251212Z","logger":"scheduler","caller":"scheduler/scheduler.go:328","msg":"Nomination done","replica-role":"leader","schedulingCycle":3,"entries":1,"inadmissibleEntries":0,"duration":0.00007542}
{"level":"Level(-2)","ts":"2026-11-03T04:09:52.030279695Z","logger":"scheduler","caller":"scheduler/scheduler.go:406","msg":"Attempting to schedule workload","replica-role":"leader","schedulingCycle":3,"workload":{"name":"trainjob-test-preemption-trainjob-fnt6p-dced9","namespace":"test-ns-289zf"},"clusterQueue":{"name":"cq-s4rfg"}}
{"level":"Level(-2)","ts":"2026-11-03T04:09:52.030363941Z","logger":"scheduler","caller":"scheduler/scheduler.go:1041","msg":"Workload assumed in the cache","replica-role":"leader","schedulingCycle":3,"workload":{"name":"trainjob-test-preemption-trainjob-fnt6p-dced9","namespace":"test-ns-289zf"},"clusterQueue":{"name":"cq-s4rfg"}}
{"level":"Level(-2)","ts":"2026-11-03T04:09:52.030392457Z","logger":"scheduler","caller":"scheduler/scheduler.go:362","msg":"Workload processing done","replica-role":"leader","schedulingCycle":3,"duration":0.000118596}
{"level":"Level(-2)","ts":"2026-11-03T04:09:52.030409087Z","logger":"scheduler","caller":"scheduler/scheduler.go:297","msg":"Scheduling cycle complete","replica-role":"leader","schedulingCycle":3,"duration":35.64364281}
{"level":"Level(-2)","ts":"2026-11-03T04:09:52.030427007Z","logger":"scheduler","caller":"scheduler/scheduler.go:295","msg":"Scheduling cycle starts","replica-role":"leader","schedulingCycle":4}
{"level":"Level(-2)","ts":"2026-11-03T04:09:52.141754005Z","logger":"scheduler","caller":"scheduler/scheduler.go:1000","msg":"Workload successfully admitted and assigned flavors","replica-role":"leader","schedulingCycle":3,"workload":{"name":"trainjob-test-preemption-trainjob-fnt6p-dced9","namespace":"test-ns-289zf"},"clusterQueue":{"name":"cq-s4rfg"},"assignments":[{"name":"node","count":1}]}
{"level":"debug","ts":"2026-11-03T04:09:52.141795861Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-test-preemption-trainjob-fnt6p-dced9","namespace":"test-ns-289zf"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"QuotaReserved","action":"QuotaReserved","note":"Quota reserved in ClusterQueue cq-s4rfg, wait time since queued was 1s"}
{"level":"debug","ts":"2026-11-03T04:09:52.141828505Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-test-preemption-trainjob-fnt6p-dced9","namespace":"test-ns-289zf"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"Admitted","action":"Admitted","note":"Admitted by ClusterQueue cq-s4rfg, wait time since reservation was 0s"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:52.141817538Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-289zf","name":"trainjob-test-preemption-trainjob-fnt6p-dced9","reconcileID":"df7c52aa-a86a-405d-a0e8-dd28471b41e1"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:52.141845518Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-fnt6p-dced9","namespace":"test-ns-289zf"},"queue":"lq-7qwch","status":"admitted","prevStatus":"pending","clusterQueue":"cq-s4rfg"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:52.141875022Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-289zf","name":"trainjob-test-preemption-trainjob-fnt6p-dced9","reconcileID":"df7c52aa-a86a-405d-a0e8-dd28471b41e1","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-03T04:09:52.142020516Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-289zf","name":"trainjob-test-preemption-trainjob-fnt6p-dced9","reconcileID":"a2acd408-a2e2-4669-9852-ff0cd4d00ef5"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:52.189180352Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"d1d4a6e5-a2b6-4f18-8a27-a31f9f8835a4","job":"test-ns-289zf/test-preemption-trainjob-fnt6p","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:52.348006912Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"d1d4a6e5-a2b6-4f18-8a27-a31f9f8835a4","job":"test-ns-289zf/test-preemption-trainjob-fnt6p","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-3)","ts":"2026-11-03T04:09:52.457163497Z","logger":"trainjob","caller":"jobframework/reconciler.go:594","msg":"Unsuspending job","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"d1d4a6e5-a2b6-4f18-8a27-a31f9f8835a4","job":"test-ns-289zf/test-preemption-trainjob-fnt6p","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-preemption-trainjob-fnt6p\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-3)","ts":"2026-11-03T04:09:52.457255166Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"d1d4a6e5-a2b6-4f18-8a27-a31f9f8835a4","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-preemption-trainjob-fnt6p\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:52.457373275Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"557bbc1e-9d9f-413a-9051-80020b4e9420","job":"test-ns-289zf/test-preemption-trainjob-fnt6p","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:52.616195157Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"557bbc1e-9d9f-413a-9051-80020b4e9420","job":"test-ns-289zf/test-preemption-trainjob-fnt6p","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-3)","ts":"2026-11-03T04:09:52.725023281Z","logger":"trainjob","caller":"jobframework/reconciler.go:594","msg":"Unsuspending job","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"557bbc1e-9d9f-413a-9051-80020b4e9420","job":"test-ns-289zf/test-preemption-trainjob-fnt6p","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-preemption-trainjob-fnt6p\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-3)","ts":"2026-11-03T04:09:52.7251249Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"557bbc1e-9d9f-413a-9051-80020b4e9420","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-preemption-trainjob-fnt6p\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:52.725221693Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"b57e147a-21b2-427b-9597-c724707e76b4","job":"test-ns-289zf/test-preemption-trainjob-fnt6p","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:52.884902159Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"b57e147a-21b2-427b-9597-c724707e76b4","job":"test-ns-289zf/test-preemption-trainjob-fnt6p","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"info","ts":"2026-11-03T04:09:52.995233727Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-preemption-trainjob-fnt6p","namespace":"test-ns-289zf"},"namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"8e0a93ac-032c-485b-bd91-3f6dc2485b5e","replica-role":"leader"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:53.030597726Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-289zf","name":"lq-7qwch","reconcileID":"6d47b776-9057-4757-abcc-1ecca2a4fe79"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:53.030596476Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-s4rfg","reconcileID":"a8ef9ab4-8c18-4777-b75b-4a410851f691"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:53.089435954Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-7qwch","namespace":"test-ns-289zf"}}
{"level":"Level(-2)","ts":"2026-11-03T04:09:53.089551718Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-289zf","name":"lq-7qwch","reconcileID":"95cee8ab-28f6-4a48-a9d7-fe7267021138"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:53.092188821Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-s4rfg"}}
{"level":"Level(-2)","ts":"2026-11-03T04:09:53.092346263Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-s4rfg","reconcileID":"099820d7-c146-4388-b783-89890b40da3f"}
{"level":"info","ts":"2026-11-03T04:09:53.162386906Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-preemption-trainjob-fnt6p","namespace":"test-ns-289zf"},"namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"349db274-7f67-4741-bb26-c4c7ed51b36c","replica-role":"leader"}
{"level":"debug","ts":"2026-11-03T04:09:53.219097095Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"test-preemption-trainjob-fnt6p","namespace":"test-ns-289zf"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"Started","action":"Started","note":"Admitted by clusterQueue cq-s4rfg"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:53.219114969Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"84c78105-725c-4e5b-97d2-3a589bb977b5","job":"test-ns-289zf/test-preemption-trainjob-fnt6p","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:53.379023946Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"bc7b4de6-8b9a-4c7d-ba4d-4b24b5a0af51","job":"test-ns-289zf/test-preemption-trainjob-fnt6p","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:53.5389659Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"fe800f81-3ad9-40c3-903a-c03f3dd226f8","job":"test-ns-289zf/test-preemption-trainjob-fnt6p","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:53.698813132Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"526de645-ee52-4ab3-8c3d-af25b0b2ff79","job":"test-ns-289zf/test-preemption-trainjob-fnt6p","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:53.858687698Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"b9f9281d-ad90-4ad9-bdf8-9cb33198c9b6","job":"test-ns-289zf/test-preemption-trainjob-fnt6p","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:54.570681958Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-289zf","name":"trainjob-test-preemption-trainjob-fnt6p-dced9","reconcileID":"494052e3-86ea-43e6-9bbd-4bd26cb9cc37"}
{"level":"Level(-2)","ts":"2026-11-03T04:10:21.417694239Z","logger":"v1_pod","caller":"workload/workload.go:1810","msg":"Workload is orphaned; finishing to release quota","replica-role":"leader","namespace":"test-ns-pj9cs","name":"dw-deployment-rnsbp-d7d46fb8-777pw","reconcileID":"a59ca778-e215-48b0-b5f3-c89089d5f46b","job":"test-ns-pj9cs/dw-deployment-rnsbp-d7d46fb8-777pw","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-03T04:10:21.418334789Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-pj9cs","name":"pod-dw-deployment-rnsbp-d7d46fb8-777pw-383b1","reconcileID":"401c1cb4-81fc-4cb7-9131-834005bd2e87"}
{"level":"Level(-2)","ts":"2026-11-03T04:10:21.418359741Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"pod-dw-deployment-rnsbp-d7d46fb8-777pw-383b1","namespace":"test-ns-pj9cs"},"queue":"lq-gntg5","status":"admitted","clusterQueue":"cq-m2r6r"}
{"level":"Level(-2)","ts":"2026-11-03T04:10:21.418405768Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-pj9cs","name":"pod-dw-deployment-rnsbp-d7d46fb8-777pw-383b1","reconcileID":"401c1cb4-81fc-4cb7-9131-834005bd2e87","isDeleted":true}
{"level":"Level(-2)","ts":"2026-11-03T04:10:21.418598163Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-pj9cs","name":"pod-dw-deployment-rnsbp-d7d46fb8-777pw-383b1","reconcileID":"2b0de1c4-5d49-4946-9d23-c39b975beb66"}
{"level":"Level(-3)","ts":"2026-11-03T04:10:21.476978942Z","logger":"v1_pod","caller":"workload/workload.go:1814","msg":"Failed to finish Workload","replica-role":"leader","namespace":"test-ns-pj9cs","name":"dw-deployment-rnsbp-d7d46fb8-777pw","reconcileID":"a59ca778-e215-48b0-b5f3-c89089d5f46b","job":"test-ns-pj9cs/dw-deployment-rnsbp-d7d46fb8-777pw","gvk":"/v1, Kind=Pod","error":"Operation cannot be fulfilled on workloads.kueue.x-k8s.io \"pod-dw-deployment-rnsbp-d7d46fb8-777pw-383b1\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-3)","ts":"2026-11-03T04:10:21.47708459Z","logger":"v1_pod","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-pj9cs","name":"dw-deployment-rnsbp-d7d46fb8-777pw","reconcileID":"a59ca778-e215-48b0-b5f3-c89089d5f46b","error":"Operation cannot be fulfilled on workloads.kueue.x-k8s.io \"pod-dw-deployment-rnsbp-d7d46fb8-777pw-383b1\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-11-03T04:10:21.598660661Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1312","msg":"Workload delete event","replica-role":"leader","workload":{"name":"pod-dw-deployment-rnsbp-d7d46fb8-777pw-383b1","namespace":"test-ns-pj9cs"},"queue":"lq-gntg5","status":"admitted"}
{"level":"Level(-2)","ts":"2026-11-03T04:10:21.598727195Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-pj9cs","name":"pod-dw-deployment-rnsbp-d7d46fb8-777pw-383b1","reconcileID":"9762fbf9-3826-421f-8ce3-b06e251c224f"}
{"level":"Level(-2)","ts":"2026-11-03T04:10:21.598782184Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-pj9cs","name":"pod-dw-deployment-rnsbp-d7d46fb8-777pw-383b1","reconcileID":"9762fbf9-3826-421f-8ce3-b06e251c224f","isDeleted":true}
{"level":"Level(-2)","ts":"2026-11-03T04:10:21.787254319Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-pj9cs","name":"pod-dw-deployment-rnsbp-d7d46fb8-777pw-383b1","reconcileID":"30a3b5f8-f689-4bf3-8adc-c34b20bc8e9e"}
{"level":"Level(-2)","ts":"2026-11-03T04:10:22.419436566Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-pj9cs","name":"lq-gntg5","reconcileID":"0335f5ea-7cd7-4063-b955-8db00c8af77b"}
{"level":"Level(-2)","ts":"2026-11-03T04:10:22.419476684Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-m2r6r","reconcileID":"6636023e-9480-485f-afd8-de25edba724b"}
{"level":"Level(-2)","ts":"2026-11-03T04:10:22.477807549Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-gntg5","namespace":"test-ns-pj9cs"}}
{"level":"Level(-2)","ts":"2026-11-03T04:10:22.477930397Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-pj9cs","name":"lq-gntg5","reconcileID":"de5f7c2e-c590-4cd1-bff1-768f4d3aab20"}
{"level":"Level(-2)","ts":"2026-11-03T04:10:22.530843841Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:376","msg":"ClusterQueue delete event","replica-role":"leader","clusterQueue":{"name":"cq-m2r6r"}}
{"level":"Level(-2)","ts":"2026-11-03T04:10:22.530926836Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:385","msg":"Cleared resource metrics for deleted ClusterQueue.","replica-role":"leader","clusterQueue":{"name":"cq-m2r6r"}}
{"level":"Level(-2)","ts":"2026-11-03T04:10:22.530943865Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-pj9cs","name":"lq-gntg5","reconcileID":"dab7c3b0-5aa3-4256-973a-178e15967e4e"}
{"level":"Level(-2)","ts":"2026-11-03T04:10:22.531080488Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-fkql6","reconcileID":"59288bc6-f89d-45d2-a560-e3e0d1c6761c"}
{"level":"Level(-2)","ts":"2026-11-03T04:10:22.586534043Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-gntg5","namespace":"test-ns-pj9cs"}}
{"level":"Level(-2)","ts":"2026-11-03T04:10:22.586637493Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-pj9cs","name":"lq-gntg5","reconcileID":"c76ffc64-7a47-4f7b-9f49-a85570af5094"}
{"level":"Level(-2)","ts":"2026-11-03T04:10:22.634670415Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:159","msg":"ResourceFlavor delete event","replica-role":"leader","resourceFlavor":{"name":"rf-fkql6"}}
{"level":"Level(-2)","ts":"2026-11-03T04:10:33.646429723Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:273","msg":"LocalQueue delete event","replica-role":"leader","localQueue":{"name":"lq-gntg5","namespace":"test-ns-pj9cs"}}
{"level":"Level(-2)","ts":"2026-11-03T04:11:11.745050938Z","logger":"tas-node-reconciler","caller":"tas/node_controller.go:136","msg":"Reconcile Node","replica-role":"leader","name":"ip-10-0-137-46.us-west-2.compute.internal","reconcileID":"5cf4fdd5-6690-48a0-826b-5a4b006f3564"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:08.175306026Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-fnt6p-dced9","namespace":"test-ns-289zf"},"queue":"lq-7qwch","status":"admitted","clusterQueue":"cq-s4rfg"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:08.175389527Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1381","msg":"Workload will not be queued because the workload is not active","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-fnt6p-dced9","namespace":"test-ns-289zf"},"queue":"lq-7qwch","status":"admitted","clusterQueue":"cq-s4rfg"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:08.175310459Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-289zf","name":"trainjob-test-preemption-trainjob-fnt6p-dced9","reconcileID":"d3964e38-fc2e-4539-919c-0153bfa38f11"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:08.175427401Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"f0a3746c-4843-4fe8-b397-832359899ee9","job":"test-ns-289zf/test-preemption-trainjob-fnt6p","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:08.175444615Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-289zf","name":"trainjob-test-preemption-trainjob-fnt6p-dced9","reconcileID":"d3964e38-fc2e-4539-919c-0153bfa38f11","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-03T04:15:08.175519687Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-289zf","name":"trainjob-test-preemption-trainjob-fnt6p-dced9","reconcileID":"3a57ca14-6cdc-43c7-b90a-c37931d8c2be"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:08.292766615Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-289zf","name":"trainjob-test-preemption-trainjob-fnt6p-dced9","reconcileID":"f30d0509-9881-46a7-8f23-2d95bd5edf02"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:08.29282804Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-289zf","name":"trainjob-test-preemption-trainjob-fnt6p-dced9","reconcileID":"f30d0509-9881-46a7-8f23-2d95bd5edf02","isDeleted":false}
{"level":"debug","ts":"2026-11-03T04:15:08.292739756Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-test-preemption-trainjob-fnt6p-dced9","namespace":"test-ns-289zf"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"EvictedDueToDeactivated","action":"EvictedDueToDeactivated","note":"The workload is deactivated"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:08.292815153Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-fnt6p-dced9","namespace":"test-ns-289zf"},"queue":"lq-7qwch","status":"admitted","clusterQueue":"cq-s4rfg"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:08.292874619Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1381","msg":"Workload will not be queued because the workload is not active","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-fnt6p-dced9","namespace":"test-ns-289zf"},"queue":"lq-7qwch","status":"admitted","clusterQueue":"cq-s4rfg"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:08.292986807Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-289zf","name":"trainjob-test-preemption-trainjob-fnt6p-dced9","reconcileID":"15182a3b-4f51-4d46-8323-b96eec1bfbd0"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:08.455268039Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"fde43ee8-ee9d-4570-8baa-24ea043e6c74","job":"test-ns-289zf/test-preemption-trainjob-fnt6p","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:08.851244424Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-289zf","name":"trainjob-test-preemption-trainjob-fnt6p-dced9","reconcileID":"0bf86679-2f72-40bb-b7a3-f180907dfff2"}
{"level":"error","ts":"2026-11-03T04:15:09.024511939Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"fde43ee8-ee9d-4570-8baa-24ea043e6c74","error":"jobs are still active","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"debug","ts":"2026-11-03T04:15:09.024581818Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"test-preemption-trainjob-fnt6p","namespace":"test-ns-289zf"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"Stopped","action":"Stopped","note":"The workload is deactivated"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:09.024673008Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"a52c5e5b-3e77-4db4-84d1-1c6e57120d1d","job":"test-ns-289zf/test-preemption-trainjob-fnt6p","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:09.175947332Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-289zf","name":"lq-7qwch","reconcileID":"045c6ec7-8405-4f5e-bf52-086f9c43ebd5"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:09.175974596Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-s4rfg","reconcileID":"0c9ce692-53a4-4af8-9ffe-bd24bf178d39"}
{"level":"error","ts":"2026-11-03T04:15:09.184400832Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"a52c5e5b-3e77-4db4-84d1-1c6e57120d1d","error":"jobs are still active","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:09.184520446Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"2323f6cd-e90e-4284-9d81-a85d0de25025","job":"test-ns-289zf/test-preemption-trainjob-fnt6p","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:09.235002318Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-7qwch","namespace":"test-ns-289zf"}}
{"level":"Level(-2)","ts":"2026-11-03T04:15:09.235173415Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-289zf","name":"lq-7qwch","reconcileID":"f020feee-22cf-4667-9da1-779e6aff727f"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:09.237907221Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-s4rfg"}}
{"level":"Level(-2)","ts":"2026-11-03T04:15:09.238233153Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-s4rfg","reconcileID":"162c7bb4-f0b7-489a-8081-704d2274c132"}
{"level":"error","ts":"2026-11-03T04:15:09.343599905Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"2323f6cd-e90e-4284-9d81-a85d0de25025","error":"jobs are still active","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:09.343767254Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"7cccc31f-bddd-4891-a3fd-b480679b5c5f","job":"test-ns-289zf/test-preemption-trainjob-fnt6p","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-11-03T04:15:09.50356035Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"7cccc31f-bddd-4891-a3fd-b480679b5c5f","error":"jobs are still active","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:09.5037104Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"ec973861-575d-4e19-8f86-6aeb56f0cb4a","job":"test-ns-289zf/test-preemption-trainjob-fnt6p","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-11-03T04:15:09.663310677Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"ec973861-575d-4e19-8f86-6aeb56f0cb4a","error":"jobs are still active","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:09.663445969Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"477b0490-c548-4f44-9a85-ecfcf21a892f","job":"test-ns-289zf/test-preemption-trainjob-fnt6p","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-11-03T04:15:09.823329923Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"477b0490-c548-4f44-9a85-ecfcf21a892f","error":"jobs are still active","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:09.823457808Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"813aed9f-6db4-401a-8a97-8168c5112e74","job":"test-ns-289zf/test-preemption-trainjob-fnt6p","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-11-03T04:15:09.983392367Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"813aed9f-6db4-401a-8a97-8168c5112e74","error":"jobs are still active","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:09.983522014Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"5a11d105-9059-4758-a123-9ee96170a900","job":"test-ns-289zf/test-preemption-trainjob-fnt6p","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.257658383Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-s4rfg"}}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.257766471Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-s4rfg","reconcileID":"5341abeb-44c2-49f3-a22a-22cea2588410"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.257878311Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-289zf","name":"trainjob-test-preemption-trainjob-fnt6p-dced9","reconcileID":"b49cd8e0-da26-49bd-9ea8-acecad3a95e2"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.274277942Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:171","msg":"ResourceFlavor update event","replica-role":"leader","resourceFlavor":{"name":"rf-j9bhm"}}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.274420073Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-j9bhm","reconcileID":"e6552b0f-3bfa-46c9-9a10-ce274ea2700f"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.420462238Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-fnt6p-dced9","namespace":"test-ns-289zf"},"queue":"lq-7qwch","status":"pending","prevStatus":"admitted","prevClusterQueue":"cq-s4rfg"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.420517776Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-289zf","name":"trainjob-test-preemption-trainjob-fnt6p-dced9","reconcileID":"e92c135d-a00b-42e9-810f-ee8f8cdc3877"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.420554834Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1381","msg":"Workload will not be queued because the workload is not active","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-fnt6p-dced9","namespace":"test-ns-289zf"},"queue":"lq-7qwch","status":"pending","prevStatus":"admitted","prevClusterQueue":"cq-s4rfg"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.420568906Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-289zf","name":"trainjob-test-preemption-trainjob-fnt6p-dced9","reconcileID":"e92c135d-a00b-42e9-810f-ee8f8cdc3877","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.420440682Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"90649874-d9f2-4339-bd51-8eb9a58fd4b1","job":"test-ns-289zf/test-preemption-trainjob-fnt6p","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.420688808Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-289zf","name":"trainjob-test-preemption-trainjob-fnt6p-dced9","reconcileID":"3b46b8fd-d659-41ea-80ae-0bef301a83a9"}
{"level":"error","ts":"2026-11-03T04:15:10.472879636Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"90649874-d9f2-4339-bd51-8eb9a58fd4b1","job":"test-ns-289zf/test-preemption-trainjob-fnt6p","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-fnt6p\" is forbidden: unable to create new content in namespace test-ns-289zf because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-03T04:15:10.472957802Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"90649874-d9f2-4339-bd51-8eb9a58fd4b1","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-fnt6p\" is forbidden: unable to create new content in namespace test-ns-289zf because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.473077883Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"3bf76110-5bb5-4a37-9be6-26061e18b1b6","job":"test-ns-289zf/test-preemption-trainjob-fnt6p","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.487402595Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:376","msg":"ClusterQueue delete event","replica-role":"leader","clusterQueue":{"name":"cq-s4rfg"}}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.487477535Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:385","msg":"Cleared resource metrics for deleted ClusterQueue.","replica-role":"leader","clusterQueue":{"name":"cq-s4rfg"}}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.487491094Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-289zf","name":"lq-7qwch","reconcileID":"7f75990c-2428-4c99-a59a-ba13757d4431"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.48758134Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-j9bhm","reconcileID":"c0860da6-7a7a-4403-8ec3-16870684e068"}
{"level":"error","ts":"2026-11-03T04:15:10.525254142Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"3bf76110-5bb5-4a37-9be6-26061e18b1b6","job":"test-ns-289zf/test-preemption-trainjob-fnt6p","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-fnt6p\" is forbidden: unable to create new content in namespace test-ns-289zf because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-03T04:15:10.525340277Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"3bf76110-5bb5-4a37-9be6-26061e18b1b6","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-fnt6p\" is forbidden: unable to create new content in namespace test-ns-289zf because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.535362875Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-289zf","name":"trainjob-test-preemption-trainjob-fnt6p-dced9","reconcileID":"2315aca0-dc38-4c2d-b107-a3e0b9e10c0a"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.535457037Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"44d06ef9-faf8-42d0-a49f-a33a630f33d8","job":"test-ns-289zf/test-preemption-trainjob-fnt6p","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.535509068Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-fnt6p-dced9","namespace":"test-ns-289zf"},"queue":"lq-7qwch","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.535560998Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1381","msg":"Workload will not be queued because the workload is not active","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-fnt6p-dced9","namespace":"test-ns-289zf"},"queue":"lq-7qwch","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.535464824Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-289zf","name":"trainjob-test-preemption-trainjob-fnt6p-dced9","reconcileID":"8ad95a15-fd88-4758-a859-69671556d2f0"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.535612021Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-289zf","name":"trainjob-test-preemption-trainjob-fnt6p-dced9","reconcileID":"8ad95a15-fd88-4758-a859-69671556d2f0","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.544864952Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-7qwch","namespace":"test-ns-289zf"}}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.545084905Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-289zf","name":"lq-7qwch","reconcileID":"ec8acb2d-6760-446f-8a45-0ed542a42344"}
{"level":"error","ts":"2026-11-03T04:15:10.587690059Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"44d06ef9-faf8-42d0-a49f-a33a630f33d8","job":"test-ns-289zf/test-preemption-trainjob-fnt6p","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-fnt6p\" is forbidden: unable to create new content in namespace test-ns-289zf because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-03T04:15:10.587765409Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"44d06ef9-faf8-42d0-a49f-a33a630f33d8","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-fnt6p\" is forbidden: unable to create new content in namespace test-ns-289zf because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.587858332Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"46c0bfee-e4d1-4cad-843c-22cd70174566","job":"test-ns-289zf/test-preemption-trainjob-fnt6p","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-11-03T04:15:10.639968694Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"46c0bfee-e4d1-4cad-843c-22cd70174566","job":"test-ns-289zf/test-preemption-trainjob-fnt6p","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-fnt6p\" is forbidden: unable to create new content in namespace test-ns-289zf because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-03T04:15:10.640040342Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"46c0bfee-e4d1-4cad-843c-22cd70174566","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-fnt6p\" is forbidden: unable to create new content in namespace test-ns-289zf because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.649431035Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-289zf","name":"trainjob-test-preemption-trainjob-fnt6p-dced9","reconcileID":"094630b8-93b1-4f84-a3b8-bbdf74b2d347"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.649460631Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-289zf","name":"trainjob-test-preemption-trainjob-fnt6p-dced9","reconcileID":"9915c3ce-954c-416e-a9c7-276b8a62d305"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.649423331Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-fnt6p-dced9","namespace":"test-ns-289zf"},"queue":"lq-7qwch","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.649507438Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-289zf","name":"trainjob-test-preemption-trainjob-fnt6p-dced9","reconcileID":"9915c3ce-954c-416e-a9c7-276b8a62d305","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.649512658Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1381","msg":"Workload will not be queued because the workload is not active","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-fnt6p-dced9","namespace":"test-ns-289zf"},"queue":"lq-7qwch","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.649540674Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"e5c85654-49e3-4254-ae0f-30a0521bbe8d","job":"test-ns-289zf/test-preemption-trainjob-fnt6p","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.649567148Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-289zf","name":"trainjob-test-preemption-trainjob-fnt6p-dced9","reconcileID":"0eb5984d-702c-4e5f-852d-344a6c18afde"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.700648066Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:159","msg":"ResourceFlavor delete event","replica-role":"leader","resourceFlavor":{"name":"rf-j9bhm"}}
{"level":"error","ts":"2026-11-03T04:15:10.701680666Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"e5c85654-49e3-4254-ae0f-30a0521bbe8d","job":"test-ns-289zf/test-preemption-trainjob-fnt6p","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-fnt6p\" is forbidden: unable to create new content in namespace test-ns-289zf because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-03T04:15:10.701745734Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"e5c85654-49e3-4254-ae0f-30a0521bbe8d","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-fnt6p\" is forbidden: unable to create new content in namespace test-ns-289zf because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.748107793Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:141","msg":"ResourceFlavor create event","replica-role":"leader","resourceFlavor":{"name":"rf-58n95"}}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.748222657Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-58n95","reconcileID":"0d6f624c-c7f7-4013-97d3-9d3872bbbcaa"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.782203166Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"34fd70a1-4f7e-4cd7-9306-943e9db7c82e","job":"test-ns-289zf/test-preemption-trainjob-fnt6p","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-11-03T04:15:10.834516284Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"34fd70a1-4f7e-4cd7-9306-943e9db7c82e","job":"test-ns-289zf/test-preemption-trainjob-fnt6p","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-fnt6p\" is forbidden: unable to create new content in namespace test-ns-289zf because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-03T04:15:10.834590913Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"34fd70a1-4f7e-4cd7-9306-943e9db7c82e","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-fnt6p\" is forbidden: unable to create new content in namespace test-ns-289zf because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.976726545Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:351","msg":"ClusterQueue create event","replica-role":"leader","clusterQueue":{"name":"cq-vkftm"}}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.976934116Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-vkftm","reconcileID":"b3ce9481-3ab1-4b9d-a0b9-6f08a2c80d0e"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.994288297Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:240","msg":"LocalQueue create event","replica-role":"leader","localQueue":{"name":"lq-z6n2r","namespace":"test-ns-xglkq"}}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.994435937Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-xglkq","name":"lq-z6n2r","reconcileID":"12f1a1ca-4dfc-4b0d-a3a1-e7a60403f427"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.994740587Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"6a30c48a-aa2d-41ec-b028-f1b0b35caf7b","job":"test-ns-289zf/test-preemption-trainjob-fnt6p","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:11.035804445Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-vkftm"}}
{"level":"Level(-2)","ts":"2026-11-03T04:15:11.03598756Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-vkftm","reconcileID":"d8b53ec5-6229-4073-bf4d-ebc636cc7845"}
{"level":"error","ts":"2026-11-03T04:15:11.046841597Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"6a30c48a-aa2d-41ec-b028-f1b0b35caf7b","job":"test-ns-289zf/test-preemption-trainjob-fnt6p","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-fnt6p\" is forbidden: unable to create new content in namespace test-ns-289zf because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-03T04:15:11.046917108Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"6a30c48a-aa2d-41ec-b028-f1b0b35caf7b","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-fnt6p\" is forbidden: unable to create new content in namespace test-ns-289zf because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:11.052191229Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-z6n2r","namespace":"test-ns-xglkq"}}
{"level":"Level(-2)","ts":"2026-11-03T04:15:11.052194635Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-xglkq","name":"lq-z6n2r","reconcileID":"b53737c8-9f39-483c-a85b-c2855bc31771"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:11.108332553Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-z6n2r","namespace":"test-ns-xglkq"}}
{"level":"Level(-2)","ts":"2026-11-03T04:15:11.108415351Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-xglkq","name":"lq-z6n2r","reconcileID":"aebb91a2-4454-4aa4-92ff-0be88c54b0f7"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:11.108504838Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-xglkq","name":"lq-z6n2r","reconcileID":"b84508c3-d417-475a-9412-be7b98b30ab0"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:11.285830616Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-xglkq","name":"test-runtimepatches-trainjob-zx4cq","reconcileID":"2eb6a60f-c498-4209-8cf5-2d17a1d4edba","job":"test-ns-xglkq/test-runtimepatches-trainjob-zx4cq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:11.605628077Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1276","msg":"Workload create event","replica-role":"leader","workload":{"name":"trainjob-test-runtimepatches-trainjob-zx4cq-b2e76","namespace":"test-ns-xglkq"},"queue":"lq-z6n2r","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:11.60567979Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-xglkq","name":"trainjob-test-runtimepatches-trainjob-zx4cq-b2e76","reconcileID":"2b7a3d73-9671-489d-ba42-ce5760e0f4d7"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:11.605729159Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-xglkq","name":"trainjob-test-runtimepatches-trainjob-zx4cq-b2e76","reconcileID":"2b7a3d73-9671-489d-ba42-ce5760e0f4d7","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-03T04:15:11.60581039Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"b213259e-e6ba-44fb-b781-b3ca7ca3cb4e","job":"test-ns-289zf/test-preemption-trainjob-fnt6p","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"debug","ts":"2026-11-03T04:15:11.605831022Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"test-runtimepatches-trainjob-zx4cq","namespace":"test-ns-xglkq"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"CreatedWorkload","action":"CreatedWorkload","note":"Created Workload: test-ns-xglkq/trainjob-test-runtimepatches-trainjob-zx4cq-b2e76"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:11.605846695Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-xglkq","name":"trainjob-test-runtimepatches-trainjob-zx4cq-b2e76","reconcileID":"43e09805-0872-4e40-9d20-092014a42a57"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:11.60585771Z","logger":"scheduler","caller":"scheduler/scheduler.go:308","msg":"Obtained heads","replica-role":"leader","schedulingCycle":4,"headCount":1,"waitDuration":319.575429744}
{"level":"Level(-2)","ts":"2026-11-03T04:15:11.60591073Z","logger":"scheduler","caller":"scheduler/scheduler.go:323","msg":"Snapshot taken","replica-role":"leader","schedulingCycle":4,"duration":0.000011494}
{"level":"Level(-2)","ts":"2026-11-03T04:15:11.606014521Z","logger":"scheduler","caller":"scheduler/scheduler.go:328","msg":"Nomination done","replica-role":"leader","schedulingCycle":4,"entries":1,"inadmissibleEntries":0,"duration":0.000092604}
{"level":"Level(-2)","ts":"2026-11-03T04:15:11.606045657Z","logger":"scheduler","caller":"scheduler/scheduler.go:406","msg":"Attempting to schedule workload","replica-role":"leader","schedulingCycle":4,"workload":{"name":"trainjob-test-runtimepatches-trainjob-zx4cq-b2e76","namespace":"test-ns-xglkq"},"clusterQueue":{"name":"cq-vkftm"}}
{"level":"Level(-2)","ts":"2026-11-03T04:15:11.606176768Z","logger":"scheduler","caller":"scheduler/scheduler.go:1041","msg":"Workload assumed in the cache","replica-role":"leader","schedulingCycle":4,"workload":{"name":"trainjob-test-runtimepatches-trainjob-zx4cq-b2e76","namespace":"test-ns-xglkq"},"clusterQueue":{"name":"cq-vkftm"}}
{"level":"Level(-2)","ts":"2026-11-03T04:15:11.606205874Z","logger":"scheduler","caller":"scheduler/scheduler.go:362","msg":"Workload processing done","replica-role":"leader","schedulingCycle":4,"duration":0.000166549}
{"level":"Level(-2)","ts":"2026-11-03T04:15:11.60621891Z","logger":"scheduler","caller":"scheduler/scheduler.go:297","msg":"Scheduling cycle complete","replica-role":"leader","schedulingCycle":4,"duration":319.575791808}
{"level":"Level(-2)","ts":"2026-11-03T04:15:11.606232371Z","logger":"scheduler","caller":"scheduler/scheduler.go:295","msg":"Scheduling cycle starts","replica-role":"leader","schedulingCycle":5}
{"level":"Level(-2)","ts":"2026-11-03T04:15:11.650465254Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-289zf","name":"lq-7qwch","reconcileID":"e38b0b53-2144-4741-baa6-7f5c1eaf3dd3"}
{"level":"error","ts":"2026-11-03T04:15:11.657890337Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"b213259e-e6ba-44fb-b781-b3ca7ca3cb4e","job":"test-ns-289zf/test-preemption-trainjob-fnt6p","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-fnt6p\" is forbidden: unable to create new content in namespace test-ns-289zf because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-03T04:15:11.657959805Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"b213259e-e6ba-44fb-b781-b3ca7ca3cb4e","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-fnt6p\" is forbidden: unable to create new content in namespace test-ns-289zf because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:11.658049575Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-xglkq","name":"test-runtimepatches-trainjob-zx4cq","reconcileID":"b4bc77cd-344b-4f8c-8262-4748b0649c3d","job":"test-ns-xglkq/test-runtimepatches-trainjob-zx4cq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:11.718303305Z","logger":"scheduler","caller":"scheduler/scheduler.go:1000","msg":"Workload successfully admitted and assigned flavors","replica-role":"leader","schedulingCycle":4,"workload":{"name":"trainjob-test-runtimepatches-trainjob-zx4cq-b2e76","namespace":"test-ns-xglkq"},"clusterQueue":{"name":"cq-vkftm"},"assignments":[{"name":"node","flavors":{"cpu":"rf-58n95","memory":"rf-58n95"},"resourceUsage":{"cpu":"100m","memory":"128Mi"},"count":1}]}
{"level":"debug","ts":"2026-11-03T04:15:11.71832485Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-test-runtimepatches-trainjob-zx4cq-b2e76","namespace":"test-ns-xglkq"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"QuotaReserved","action":"QuotaReserved","note":"Quota reserved in ClusterQueue cq-vkftm, wait time since queued was 1s"}
{"level":"debug","ts":"2026-11-03T04:15:11.718368729Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-test-runtimepatches-trainjob-zx4cq-b2e76","namespace":"test-ns-xglkq"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"Admitted","action":"Admitted","note":"Admitted by ClusterQueue cq-vkftm, wait time since reservation was 1s"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:11.718419974Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-runtimepatches-trainjob-zx4cq-b2e76","namespace":"test-ns-xglkq"},"queue":"lq-z6n2r","status":"admitted","prevStatus":"pending","clusterQueue":"cq-vkftm"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:11.718432221Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-xglkq","name":"trainjob-test-runtimepatches-trainjob-zx4cq-b2e76","reconcileID":"ab85971e-06b4-4907-82ba-30b34431c915"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:11.718474541Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-xglkq","name":"trainjob-test-runtimepatches-trainjob-zx4cq-b2e76","reconcileID":"ab85971e-06b4-4907-82ba-30b34431c915","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-03T04:15:11.718628388Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-xglkq","name":"trainjob-test-runtimepatches-trainjob-zx4cq-b2e76","reconcileID":"88d70c33-a39d-4971-9174-3ec45e660b3e"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:11.81688882Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-xglkq","name":"test-runtimepatches-trainjob-zx4cq","reconcileID":"4b4f815b-2bc6-4b9f-9f3d-f577b35d5b13","job":"test-ns-xglkq/test-runtimepatches-trainjob-zx4cq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:11.976368166Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-xglkq","name":"test-runtimepatches-trainjob-zx4cq","reconcileID":"4b4f815b-2bc6-4b9f-9f3d-f577b35d5b13","job":"test-ns-xglkq/test-runtimepatches-trainjob-zx4cq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-3)","ts":"2026-11-03T04:15:12.196345503Z","logger":"trainjob","caller":"jobframework/reconciler.go:594","msg":"Unsuspending job","replica-role":"leader","namespace":"test-ns-xglkq","name":"test-runtimepatches-trainjob-zx4cq","reconcileID":"4b4f815b-2bc6-4b9f-9f3d-f577b35d5b13","job":"test-ns-xglkq/test-runtimepatches-trainjob-zx4cq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-runtimepatches-trainjob-zx4cq\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-3)","ts":"2026-11-03T04:15:12.196430882Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-xglkq","name":"test-runtimepatches-trainjob-zx4cq","reconcileID":"4b4f815b-2bc6-4b9f-9f3d-f577b35d5b13","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-runtimepatches-trainjob-zx4cq\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:12.19653931Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-xglkq","name":"test-runtimepatches-trainjob-zx4cq","reconcileID":"15267c0c-50f2-4d9f-88e5-fb93ef08d37c","job":"test-ns-xglkq/test-runtimepatches-trainjob-zx4cq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:12.356675236Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-xglkq","name":"test-runtimepatches-trainjob-zx4cq","reconcileID":"15267c0c-50f2-4d9f-88e5-fb93ef08d37c","job":"test-ns-xglkq/test-runtimepatches-trainjob-zx4cq","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:12.367490222Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-vkftm"}}
{"level":"Level(-2)","ts":"2026-11-03T04:15:12.36763897Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-xglkq","name":"trainjob-test-runtimepatches-trainjob-zx4cq-b2e76","reconcileID":"4aaacb94-6b80-4aa5-8f2b-35f4991c201e"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:12.36762261Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-vkftm","reconcileID":"0f5d53fd-1487-4b38-9cea-4044946abce5"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:12.384266687Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:171","msg":"ResourceFlavor update event","replica-role":"leader","resourceFlavor":{"name":"rf-58n95"}}
{"level":"Level(-2)","ts":"2026-11-03T04:15:12.384382337Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-58n95","reconcileID":"2df523af-3737-441c-9e6f-b184a394abca"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:12.429726374Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-vkftm"}}
{"level":"Level(-2)","ts":"2026-11-03T04:15:12.429838968Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-xglkq","name":"lq-z6n2r","reconcileID":"faa59e41-bb42-4459-a4d7-fa693a501c96"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:12.429876957Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-vkftm","reconcileID":"443ffcb4-3978-4002-9d93-744ba7706e95"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:12.429900612Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-xglkq","name":"trainjob-test-runtimepatches-trainjob-zx4cq-b2e76","reconcileID":"a8da6b86-7341-4cad-a3f7-c28b51a6b7f9"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:12.488214291Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-z6n2r","namespace":"test-ns-xglkq"}}
{"level":"Level(-2)","ts":"2026-11-03T04:15:12.488342673Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-xglkq","name":"lq-z6n2r","reconcileID":"3118b402-156b-4de3-9fa6-8c55182ad536"}
{"level":"debug","ts":"2026-11-03T04:15:12.684961058Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"test-runtimepatches-trainjob-zx4cq","namespace":"test-ns-xglkq"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"Started","action":"Started","note":"Admitted by clusterQueue cq-vkftm"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:12.685026503Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","reconcileID":"e38e37cd-8668-44ba-bd4b-f3c3f2d71223","job":"test-ns-289zf/test-preemption-trainjob-fnt6p","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}