{"level":"Level(-2)","ts":"2026-11-02T09:25:16.275612007Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-mqkw8","name":"e2e-test-flux","reconcileID":"b6e1b0f4-0a10-4a11-ad9a-9bf616a2fa83","job":"e2e-mqkw8/e2e-test-flux","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-mqkw8"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:16.755174549Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-mqkw8","name":"e2e-test-flux","reconcileID":"94519bd5-051e-4e1f-99f6-2ac653e51853","job":"e2e-mqkw8/e2e-test-flux","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-mqkw8"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:17.428456587Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-xcq2j","name":"e2e-update-time","reconcileID":"e36a8274-4e69-48cf-8a3f-2218da2369d7","job":"e2e-xcq2j/e2e-update-time","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-xcq2j"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:27.407120376Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-dc6xt","name":"e2e-deadline-job","reconcileID":"40192718-2ab3-4ef9-b064-16876da53d48","job":"e2e-dc6xt/e2e-deadline-job","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-dc6xt"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:27.640494966Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-dc6xt","name":"e2e-deadline-job","reconcileID":"9f4da27a-0909-4a08-8558-3d6784907530","job":"e2e-dc6xt/e2e-deadline-job","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-dc6xt"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:27.811676011Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-dc6xt","name":"e2e-deadline-job","reconcileID":"7443877c-3786-4bf6-97d5-57325bf32b0a","job":"e2e-dc6xt/e2e-deadline-job","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-dc6xt"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:27.924586781Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-dc6xt","name":"e2e-deadline-job","reconcileID":"a1fed945-383a-4dc8-95d3-0575e88c81e0","job":"e2e-dc6xt/e2e-deadline-job","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-dc6xt"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:27.94980631Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-dc6xt","name":"e2e-deadline-job","reconcileID":"4b71e256-1445-4ebd-8e6d-c7f412437e5a","job":"e2e-dc6xt/e2e-deadline-job","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-dc6xt"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:28.03987472Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-dc6xt","name":"e2e-deadline-job","reconcileID":"82583a85-c37e-440b-9106-ff1797db4dab","job":"e2e-dc6xt/e2e-deadline-job","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-dc6xt"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:31.287849131Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-dc6xt","name":"e2e-deadline-job","reconcileID":"1263f787-c08e-4d03-8f55-7fd9e5863c28","job":"e2e-dc6xt/e2e-deadline-job","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-dc6xt"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:31.403028561Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-dc6xt","name":"e2e-deadline-job","reconcileID":"3b1d0967-394e-4cce-9695-40cf7c237608","job":"e2e-dc6xt/e2e-deadline-job","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-dc6xt"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:37.114256006Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-dc6xt","name":"e2e-deadline-job","reconcileID":"ec2d1a20-6c44-4627-95c5-f01c2c12b61b","job":"e2e-dc6xt/e2e-deadline-job","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-dc6xt"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:37.180445115Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-dc6xt","name":"e2e-deadline-job","reconcileID":"ae625222-b75f-4283-abd8-1a1113edf623","job":"e2e-dc6xt/e2e-deadline-job","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-dc6xt"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:37.619549522Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-skg2l","name":"e2e-test-runtime-update","reconcileID":"e9b28d2e-99c2-4b95-841f-1e7254dcc03a","job":"e2e-skg2l/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-skg2l"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:37.844644235Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-skg2l","name":"e2e-test-runtime-update","reconcileID":"cf4bfe57-6c59-406f-a900-bfb68e943668","job":"e2e-skg2l/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-skg2l"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:38.065251982Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-skg2l","name":"e2e-test-runtime-update","reconcileID":"29f04abc-e5c5-428d-a284-7b72253cc7c3","job":"e2e-skg2l/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-skg2l"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:38.177010041Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-skg2l","name":"e2e-test-runtime-update","reconcileID":"04f920d4-106f-4bce-b1cb-0a75d9d98237","job":"e2e-skg2l/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-skg2l"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:38.179439166Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-skg2l","name":"e2e-test-runtime-update","reconcileID":"5b8c4a43-1336-40c2-b284-1c462dd6d789","job":"e2e-skg2l/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-skg2l"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:38.295839577Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-skg2l","name":"e2e-test-runtime-update","reconcileID":"ed05fe0e-c42e-488c-8afd-557262265069","job":"e2e-skg2l/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-skg2l"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:38.570351494Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-skg2l","name":"e2e-test-runtime-update","reconcileID":"39072238-8c26-4fa8-abac-f5a3183f5a53","job":"e2e-skg2l/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-skg2l"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:38.682579784Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-skg2l","name":"e2e-test-runtime-update","reconcileID":"d9ce3a33-2592-405d-ab99-f7f042f20eb7","job":"e2e-skg2l/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-skg2l"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:38.794850598Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-skg2l","name":"e2e-test-runtime-update","reconcileID":"fd9a7cbf-904e-495e-9201-584e25544185","job":"e2e-skg2l/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-skg2l"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:38.802066454Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-skg2l","name":"e2e-test-runtime-update","reconcileID":"acfa642b-9eda-40f8-ab3a-73153229bd31","job":"e2e-skg2l/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-skg2l"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:38.861701049Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-skg2l","name":"e2e-test-runtime-update","reconcileID":"a55255ec-eaab-443e-a66c-489c4751af16","job":"e2e-skg2l/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-skg2l"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:39.079843536Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-skg2l","name":"e2e-test-runtime-update","reconcileID":"a30a601d-101f-4e96-b4ea-ffee04643e86","job":"e2e-skg2l/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-skg2l"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:39.340590128Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-skg2l","name":"e2e-test-runtime-update","reconcileID":"b0136421-97a8-4aab-bac4-52043459b0fe","job":"e2e-skg2l/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-skg2l"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:39.584883294Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-skg2l","name":"e2e-test-runtime-update","reconcileID":"b21e4b2a-0cb0-48a8-809f-4ed1fbf20d2c","job":"e2e-skg2l/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-skg2l"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:39.698404702Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-skg2l","name":"e2e-test-runtime-update","reconcileID":"7e91e03b-4dc5-44b2-a355-3e1b7e3bb230","job":"e2e-skg2l/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-skg2l"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:39.761449486Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-skg2l","name":"e2e-test-runtime-update","reconcileID":"505b57a3-51de-4443-8ef6-22ae46e1ce58","job":"e2e-skg2l/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-skg2l"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:39.821563478Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-skg2l","name":"e2e-test-runtime-update","reconcileID":"e3ba178e-7061-4814-9603-e346da62a3c7","job":"e2e-skg2l/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-skg2l"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:39.892965612Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-skg2l","name":"e2e-test-runtime-update","reconcileID":"52c0af42-6ca8-45b0-884a-b11ce9f660ca","job":"e2e-skg2l/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-skg2l"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:39.923336916Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-skg2l","name":"e2e-test-runtime-update","reconcileID":"2ffd1c9a-459d-4c8b-8636-e4a9e6fd6458","job":"e2e-skg2l/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-skg2l"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:40.036464927Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-skg2l","name":"e2e-test-runtime-update","reconcileID":"5e174f06-e4ea-4626-9592-8713a99d0fb7","job":"e2e-skg2l/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-skg2l"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:40.576275576Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-vscrt","name":"e2e-test-runtime-update","reconcileID":"95f70457-f7b6-488d-b2af-2a7f99aadfad","job":"e2e-vscrt/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-vscrt"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:40.803032144Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-vscrt","name":"e2e-test-runtime-update","reconcileID":"ec96b670-f45e-4ea6-b7c4-bbc9277e019f","job":"e2e-vscrt/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-vscrt"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:40.975614477Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-vscrt","name":"e2e-test-runtime-update","reconcileID":"c7800a40-2626-4c7e-a7eb-ea9f98838f1a","job":"e2e-vscrt/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-vscrt"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:41.088363775Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-vscrt","name":"e2e-test-runtime-update","reconcileID":"ee0a44a8-edf7-45aa-8fb4-d06a6502b2fe","job":"e2e-vscrt/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-vscrt"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:41.111371838Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-vscrt","name":"e2e-test-runtime-update","reconcileID":"1e428cf7-ded5-46b5-adc5-e0a17b25a41f","job":"e2e-vscrt/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-vscrt"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:41.200627106Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-vscrt","name":"e2e-test-runtime-update","reconcileID":"97b6823b-d27f-4f55-bbc5-0f6dc996d85f","job":"e2e-vscrt/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-vscrt"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:41.522359228Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-vscrt","name":"e2e-test-runtime-update","reconcileID":"ae19db0c-dcc2-4b96-b0bb-b8ec24d6cade","job":"e2e-vscrt/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-vscrt"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:41.635641051Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-vscrt","name":"e2e-test-runtime-update","reconcileID":"bec9d8de-84fa-43b4-b6c6-c128e2653e87","job":"e2e-vscrt/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-vscrt"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:41.747029284Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-vscrt","name":"e2e-test-runtime-update","reconcileID":"5754b803-5c6a-42c7-9e3e-74c2506f20b6","job":"e2e-vscrt/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-vscrt"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:41.755817922Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-vscrt","name":"e2e-test-runtime-update","reconcileID":"dc229d24-5211-4099-bc38-83f19f000755","job":"e2e-vscrt/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-vscrt"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:41.815403746Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-vscrt","name":"e2e-test-runtime-update","reconcileID":"36ce2c98-23ed-4140-9291-d4e9640c8250","job":"e2e-vscrt/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-vscrt"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:42.027735222Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-vscrt","name":"e2e-test-runtime-update","reconcileID":"cbc3ce6a-bee2-46c2-81dd-bc09cb589013","job":"e2e-vscrt/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-vscrt"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:42.289298446Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-vscrt","name":"e2e-test-runtime-update","reconcileID":"c0911ded-26ab-44d5-914a-1855077e7bfe","job":"e2e-vscrt/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-vscrt"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:42.474670588Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-vscrt","name":"e2e-test-runtime-update","reconcileID":"6e7f5522-f056-43b1-9490-a1ea444d92f1","job":"e2e-vscrt/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-vscrt"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:42.590869759Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-vscrt","name":"e2e-test-runtime-update","reconcileID":"0b4a7ef5-dfda-4a58-95f5-8bbd7e4b9958","job":"e2e-vscrt/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-vscrt"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:42.65129455Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-vscrt","name":"e2e-test-runtime-update","reconcileID":"87c10527-0d34-4f84-8294-c79e2cb25844","job":"e2e-vscrt/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-vscrt"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:42.708824885Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-vscrt","name":"e2e-test-runtime-update","reconcileID":"3b853a0f-f1c0-4a4f-8871-9239364d6845","job":"e2e-vscrt/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-vscrt"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:42.804546018Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-vscrt","name":"e2e-test-runtime-update","reconcileID":"1eb64ad1-40ec-412c-8862-22ac880bc37b","job":"e2e-vscrt/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-vscrt"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:42.819326588Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-vscrt","name":"e2e-test-runtime-update","reconcileID":"2308dc97-fc26-4372-ae2d-649181e42989","job":"e2e-vscrt/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-vscrt"}
{"level":"Level(-2)","ts":"2026-11-02T09:25:42.934370651Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-vscrt","name":"e2e-test-runtime-update","reconcileID":"78b5b573-4a5c-4d6a-a02f-a8870b800444","job":"e2e-vscrt/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-vscrt"}
{"level":"Level(-2)","ts":"2026-11-02T09:26:38.424261545Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-fk77v","name":"test-dra-trainjob-dpc4v","reconcileID":"be78bb1b-2fdb-484a-a358-c656a69dc645","job":"test-ns-fk77v/test-dra-trainjob-dpc4v","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-fk77v"}
{"level":"Level(-2)","ts":"2026-11-02T09:26:38.657787194Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-fk77v","name":"test-dra-trainjob-dpc4v","reconcileID":"08f660fa-1d50-4cc0-8aa6-e76e2776c819","job":"test-ns-fk77v/test-dra-trainjob-dpc4v","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-fk77v"}
{"level":"Level(-2)","ts":"2026-11-02T09:26:38.881337315Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-fk77v","name":"test-dra-trainjob-dpc4v","reconcileID":"75c99b73-f3c1-4a19-b3d0-179a19e6d93b","job":"test-ns-fk77v/test-dra-trainjob-dpc4v","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-fk77v"}
{"level":"Level(-2)","ts":"2026-11-02T09:26:38.997632436Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-fk77v","name":"test-dra-trainjob-dpc4v","reconcileID":"1c19971f-e3b7-4f0c-a26d-c8ea112bef3a","job":"test-ns-fk77v/test-dra-trainjob-dpc4v","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-fk77v"}
{"level":"Level(-2)","ts":"2026-11-02T09:26:39.086794664Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-fk77v","name":"test-dra-trainjob-dpc4v","reconcileID":"b94ee686-9bbc-40d0-960c-b922daeeb4af","job":"test-ns-fk77v/test-dra-trainjob-dpc4v","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-fk77v"}
{"level":"Level(-2)","ts":"2026-11-02T09:26:39.20018895Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-fk77v","name":"test-dra-trainjob-dpc4v","reconcileID":"ac5c6cc1-155f-427e-8f95-36a77cddbb77","job":"test-ns-fk77v/test-dra-trainjob-dpc4v","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-fk77v"}
{"level":"Level(-2)","ts":"2026-11-02T09:26:43.033700661Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-6fx77","name":"test-trainjob-mzskk","reconcileID":"f77dc3ed-6808-49ae-99ca-6f989fb505ac","job":"test-ns-6fx77/test-trainjob-mzskk","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-6fx77"}
{"level":"Level(-2)","ts":"2026-11-02T09:26:43.270904812Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-6fx77","name":"test-trainjob-mzskk","reconcileID":"dfcc4984-1e03-4124-93f3-469c2e1667a1","job":"test-ns-6fx77/test-trainjob-mzskk","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-6fx77"}
{"level":"Level(-2)","ts":"2026-11-02T09:26:43.493768221Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-6fx77","name":"test-trainjob-mzskk","reconcileID":"cae15895-bf08-4a4e-ac06-36ddad79ac02","job":"test-ns-6fx77/test-trainjob-mzskk","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-6fx77"}
{"level":"Level(-2)","ts":"2026-11-02T09:26:43.578698616Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-6fx77","name":"test-trainjob-mzskk","reconcileID":"71438183-7fd4-4e30-970a-9d0d2e324c88","job":"test-ns-6fx77/test-trainjob-mzskk","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-6fx77"}
{"level":"Level(-2)","ts":"2026-11-02T09:26:43.605767541Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-6fx77","name":"test-trainjob-mzskk","reconcileID":"9893c32a-671d-4615-ab2a-7f0a9412e96b","job":"test-ns-6fx77/test-trainjob-mzskk","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-6fx77"}
{"level":"Level(-2)","ts":"2026-11-02T09:26:43.720292755Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-6fx77","name":"test-trainjob-mzskk","reconcileID":"2f53c821-5e11-4edd-8acc-955b41821fb8","job":"test-ns-6fx77/test-trainjob-mzskk","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-6fx77"}
{"level":"Level(-2)","ts":"2026-11-02T09:26:45.747626754Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-fk77v","name":"test-dra-trainjob-dpc4v","reconcileID":"e3b97286-59b2-4f8d-9d74-d7e23082ca59","job":"test-ns-fk77v/test-dra-trainjob-dpc4v","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-fk77v"}
{"level":"Level(-2)","ts":"2026-11-02T09:26:45.859848055Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-fk77v","name":"test-dra-trainjob-dpc4v","reconcileID":"508b508b-6f7a-40c0-8870-5c496eef4c02","job":"test-ns-fk77v/test-dra-trainjob-dpc4v","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-fk77v"}
{"level":"Level(-2)","ts":"2026-11-02T09:26:46.680683864Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-6fx77","name":"test-trainjob-mzskk","reconcileID":"d6456e94-4c1d-43fb-baa6-42eef549e8ad","job":"test-ns-6fx77/test-trainjob-mzskk","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-6fx77"}
{"level":"Level(-2)","ts":"2026-11-02T09:26:46.793566254Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-6fx77","name":"test-trainjob-mzskk","reconcileID":"6d0755b6-9de5-42e0-85e6-4b016f9dbb17","job":"test-ns-6fx77/test-trainjob-mzskk","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-6fx77"}
{"level":"Level(-2)","ts":"2026-11-02T09:26:50.170214673Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-fk77v","name":"test-dra-trainjob-dpc4v","reconcileID":"5b9d79f2-fa31-4940-b8f1-1aa25b695fd7","job":"test-ns-fk77v/test-dra-trainjob-dpc4v","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-fk77v"}
{"level":"Level(-2)","ts":"2026-11-02T09:26:52.715643973Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-6fx77","name":"test-trainjob-mzskk","reconcileID":"66b7747d-58a2-481c-ad84-44d676782664","job":"test-ns-6fx77/test-trainjob-mzskk","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-6fx77"}
{"level":"Level(-2)","ts":"2026-11-02T09:26:52.82829364Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-6fx77","name":"test-trainjob-mzskk","reconcileID":"9d0441bd-35ac-4641-b499-b07ea9f1a15a","job":"test-ns-6fx77/test-trainjob-mzskk","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-6fx77"}
{"level":"Level(-2)","ts":"2026-11-02T09:26:53.725440968Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-6fx77","name":"test-trainjob-mzskk","reconcileID":"f4c85a82-fcfa-4e3f-9739-2652251dd777","job":"test-ns-6fx77/test-trainjob-mzskk","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-6fx77"}
{"level":"Level(-2)","ts":"2026-11-02T09:26:53.788082207Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-6fx77","name":"test-trainjob-mzskk","reconcileID":"4527056d-8af8-4f21-a86a-53003df61fd4","job":"test-ns-6fx77/test-trainjob-mzskk","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-6fx77"}
{"level":"Level(-2)","ts":"2026-11-02T09:26:53.850329375Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-6fx77","name":"test-trainjob-mzskk","reconcileID":"48690658-64fd-4beb-8c62-79f8589c99e4","job":"test-ns-6fx77/test-trainjob-mzskk","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-6fx77"}
{"level":"Level(-2)","ts":"2026-11-02T09:26:53.964993552Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-6fx77","name":"test-trainjob-mzskk","reconcileID":"81fb77c3-df76-43dc-a058-0c5b805236a4","job":"test-ns-6fx77/test-trainjob-mzskk","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-6fx77"}
{"level":"Level(-2)","ts":"2026-11-02T09:26:54.436312728Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-9p2cq","name":"test-trainjob-7kfk6","reconcileID":"2c1c26ce-a477-46d1-a33f-7ff60f1a42b2","job":"test-ns-9p2cq/test-trainjob-7kfk6","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-9p2cq"}
{"level":"Level(-2)","ts":"2026-11-02T09:26:54.670645413Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-9p2cq","name":"test-trainjob-7kfk6","reconcileID":"4116034a-ba11-4968-a6f8-7066dd6ebc76","job":"test-ns-9p2cq/test-trainjob-7kfk6","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-9p2cq"}
{"level":"Level(-2)","ts":"2026-11-02T09:26:54.902080284Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-9p2cq","name":"test-trainjob-7kfk6","reconcileID":"f7eec008-2758-4fa9-b029-3a7d98c94498","job":"test-ns-9p2cq/test-trainjob-7kfk6","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-9p2cq"}
{"level":"Level(-2)","ts":"2026-11-02T09:26:55.001498897Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-9p2cq","name":"test-trainjob-7kfk6","reconcileID":"49b9a8bc-69e5-470a-bf89-ec5b74a8f912","job":"test-ns-9p2cq/test-trainjob-7kfk6","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-9p2cq"}
{"level":"Level(-2)","ts":"2026-11-02T09:26:55.019793539Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-9p2cq","name":"test-trainjob-7kfk6","reconcileID":"0501b8fd-900c-422c-88bc-c9eabf9aa708","job":"test-ns-9p2cq/test-trainjob-7kfk6","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-9p2cq"}
{"level":"Level(-2)","ts":"2026-11-02T09:26:55.13901077Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-9p2cq","name":"test-trainjob-7kfk6","reconcileID":"fe3a8fab-b632-4af7-940c-c5a31daa1196","job":"test-ns-9p2cq/test-trainjob-7kfk6","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-9p2cq"}
{"level":"Level(-2)","ts":"2026-11-02T09:36:16.50741875Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-9p2cq","name":"test-trainjob-7kfk6","reconcileID":"d64204df-8651-4284-bd0c-410becf838b7","job":"test-ns-9p2cq/test-trainjob-7kfk6","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-9p2cq"}
{"level":"Level(-2)","ts":"2026-11-02T09:36:16.622832054Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-9p2cq","name":"test-trainjob-7kfk6","reconcileID":"ac078ede-1a50-4c33-94e9-5cddd261122a","job":"test-ns-9p2cq/test-trainjob-7kfk6","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-9p2cq"}
{"level":"Level(-2)","ts":"2026-11-02T09:36:21.47094358Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-9p2cq","name":"test-trainjob-7kfk6","reconcileID":"708f27bb-e2f4-4f68-a8e4-43425058c167","job":"test-ns-9p2cq/test-trainjob-7kfk6","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-9p2cq"}
{"level":"Level(-2)","ts":"2026-11-02T09:36:21.585124043Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-9p2cq","name":"test-trainjob-7kfk6","reconcileID":"f7f65b1b-f9ef-4ba4-b516-dc2d2589a139","job":"test-ns-9p2cq/test-trainjob-7kfk6","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-9p2cq"}
{"level":"Level(-2)","ts":"2026-11-02T09:36:22.479382112Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-9p2cq","name":"test-trainjob-7kfk6","reconcileID":"bef76374-8445-433d-afff-d8722e5f801f","job":"test-ns-9p2cq/test-trainjob-7kfk6","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-9p2cq"}
{"level":"Level(-2)","ts":"2026-11-02T09:36:22.542479745Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-9p2cq","name":"test-trainjob-7kfk6","reconcileID":"45d0b395-ff13-4a9b-a106-80538dda25c7","job":"test-ns-9p2cq/test-trainjob-7kfk6","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-9p2cq"}
{"level":"Level(-2)","ts":"2026-11-02T09:36:22.592079836Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-9p2cq","name":"test-trainjob-7kfk6","reconcileID":"ca720d75-66ad-45fd-866c-94fbeddbc608","job":"test-ns-9p2cq/test-trainjob-7kfk6","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-9p2cq"}
{"level":"Level(-2)","ts":"2026-11-02T09:36:22.706670693Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-9p2cq","name":"test-trainjob-7kfk6","reconcileID":"b28f37e6-edea-47da-a7f1-4311393dbaad","job":"test-ns-9p2cq/test-trainjob-7kfk6","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-9p2cq"}
{"level":"Level(-2)","ts":"2026-11-02T09:36:23.542217931Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jlk8m","name":"test-trainjob-b8sx8","reconcileID":"2018ac42-f72b-4446-a1fc-4482542bc96c","job":"test-ns-jlk8m/test-trainjob-b8sx8","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jlk8m"}
{"level":"Level(-2)","ts":"2026-11-02T09:36:24.020846224Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jlk8m","name":"test-trainjob-b8sx8","reconcileID":"87b35279-5553-4e5d-803c-eb8a06aa2468","job":"test-ns-jlk8m/test-trainjob-b8sx8","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jlk8m"}
{"level":"Level(-2)","ts":"2026-11-02T09:36:24.244558258Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jlk8m","name":"test-trainjob-b8sx8","reconcileID":"4b5d244c-38e9-4a0c-a90f-aa2116ecb6b5","job":"test-ns-jlk8m/test-trainjob-b8sx8","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jlk8m"}
{"level":"Level(-2)","ts":"2026-11-02T09:36:24.358072919Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jlk8m","name":"test-trainjob-b8sx8","reconcileID":"85439914-e539-44d1-b39f-673ecf05a5e5","job":"test-ns-jlk8m/test-trainjob-b8sx8","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jlk8m"}
{"level":"Level(-2)","ts":"2026-11-02T09:36:24.574615974Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jlk8m","name":"test-trainjob-b8sx8","reconcileID":"6aae1c32-26cc-4820-b767-e75fab504450","job":"test-ns-jlk8m/test-trainjob-b8sx8","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jlk8m"}
{"level":"Level(-2)","ts":"2026-11-02T09:36:24.688507174Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jlk8m","name":"test-trainjob-b8sx8","reconcileID":"9d17c205-5e10-4e2e-8ca2-c1ce4a8fabda","job":"test-ns-jlk8m/test-trainjob-b8sx8","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jlk8m"}
{"level":"Level(-2)","ts":"2026-11-02T09:38:22.091335514Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jlk8m","name":"test-trainjob-b8sx8","reconcileID":"5296e6d2-5048-415f-a8a8-648b13835b39","job":"test-ns-jlk8m/test-trainjob-b8sx8","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jlk8m"}
{"level":"Level(-2)","ts":"2026-11-02T09:38:22.210397777Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jlk8m","name":"test-trainjob-b8sx8","reconcileID":"a2542378-8a26-4d4b-8185-73ccea1cc4c2","job":"test-ns-jlk8m/test-trainjob-b8sx8","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jlk8m"}
{"level":"Level(-2)","ts":"2026-11-02T09:38:23.114479984Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jlk8m","name":"test-trainjob-b8sx8","reconcileID":"1d5b80b4-5999-4883-9f64-3a5c565cbbb6","job":"test-ns-jlk8m/test-trainjob-b8sx8","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jlk8m"}
{"level":"Level(-2)","ts":"2026-11-02T09:38:23.227625422Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jlk8m","name":"test-trainjob-b8sx8","reconcileID":"91047313-c6f0-4e2f-afe4-ee6fdb66ea82","job":"test-ns-jlk8m/test-trainjob-b8sx8","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jlk8m"}
{"level":"Level(-2)","ts":"2026-11-02T09:38:25.304864092Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jlk8m","name":"test-trainjob-b8sx8","reconcileID":"975fa8fe-ce34-4c2b-becf-7a38977d826e","job":"test-ns-jlk8m/test-trainjob-b8sx8","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jlk8m"}
{"level":"Level(-2)","ts":"2026-11-02T09:38:25.417924941Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jlk8m","name":"test-trainjob-b8sx8","reconcileID":"fcb1af96-4d7e-408e-8861-36002d575197","job":"test-ns-jlk8m/test-trainjob-b8sx8","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jlk8m"}
{"level":"Level(-2)","ts":"2026-11-02T09:38:25.421479326Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jlk8m","name":"test-trainjob-b8sx8","reconcileID":"9703162f-001f-4c24-80f2-8e9aadc6d0bd","job":"test-ns-jlk8m/test-trainjob-b8sx8","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jlk8m"}
{"level":"Level(-2)","ts":"2026-11-02T09:38:25.531287229Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jlk8m","name":"test-trainjob-b8sx8","reconcileID":"a8cb394b-e7b8-48c7-94e5-1a4a43a49d04","job":"test-ns-jlk8m/test-trainjob-b8sx8","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jlk8m"}
{"level":"Level(-2)","ts":"2026-11-02T09:38:26.664489984Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wd7wz","name":"test-trainjob-97jgb","reconcileID":"09e5a9b9-7975-4c5e-994d-928a81147b21","job":"test-ns-wd7wz/test-trainjob-97jgb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wd7wz"}
{"level":"Level(-2)","ts":"2026-11-02T09:38:27.249313293Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wd7wz","name":"test-trainjob-97jgb","reconcileID":"2f682855-cb39-4d1c-a9db-ef5e6d8fffa0","job":"test-ns-wd7wz/test-trainjob-97jgb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wd7wz"}
{"level":"Level(-2)","ts":"2026-11-02T09:38:27.478333738Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wd7wz","name":"test-trainjob-97jgb","reconcileID":"171e4da5-d04e-4f42-a1d1-4666fc288f1e","job":"test-ns-wd7wz/test-trainjob-97jgb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wd7wz"}
{"level":"Level(-2)","ts":"2026-11-02T09:38:27.595734645Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wd7wz","name":"test-trainjob-97jgb","reconcileID":"2230f92e-f74c-402a-88bc-17c4ef4b0214","job":"test-ns-wd7wz/test-trainjob-97jgb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wd7wz"}
{"level":"Level(-2)","ts":"2026-11-02T09:38:27.812398908Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wd7wz","name":"test-trainjob-97jgb","reconcileID":"3e970ffe-8a80-420a-92c5-b2d9973922a9","job":"test-ns-wd7wz/test-trainjob-97jgb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wd7wz"}
{"level":"Level(-2)","ts":"2026-11-02T09:38:27.930863487Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wd7wz","name":"test-trainjob-97jgb","reconcileID":"4f8a392f-4849-4041-a389-6fd1b6c60a1a","job":"test-ns-wd7wz/test-trainjob-97jgb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wd7wz"}
{"level":"Level(-2)","ts":"2026-11-02T09:38:31.388802084Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-jlk8m","name":"test-trainjob-b8sx8","reconcileID":"97610bd9-a01a-4d07-be84-24a8da85ab9b","job":"test-ns-jlk8m/test-trainjob-b8sx8","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-jlk8m"}
{"level":"Level(-2)","ts":"2026-11-02T09:38:31.565872609Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-9p2cq","name":"test-trainjob-7kfk6","reconcileID":"6a2d6683-8ace-4241-b748-3cd761e9d96c","job":"test-ns-9p2cq/test-trainjob-7kfk6","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-9p2cq"}
{"level":"Level(-2)","ts":"2026-11-02T09:38:31.626768968Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-6fx77","name":"test-trainjob-mzskk","reconcileID":"6689a8fa-95d2-4dc2-9197-661c040c9555","job":"test-ns-6fx77/test-trainjob-mzskk","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-6fx77"}
{"level":"Level(-2)","ts":"2026-11-02T09:38:31.634637788Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-9p2cq","name":"test-trainjob-7kfk6","reconcileID":"ef70a1c2-bae7-45ba-a34d-7e3e55661138","job":"test-ns-9p2cq/test-trainjob-7kfk6","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-9p2cq"}
{"level":"Level(-2)","ts":"2026-11-02T09:38:31.764989512Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-6fx77","name":"test-trainjob-mzskk","reconcileID":"5dfa1eac-8dfd-4ac1-84ed-7eb982ab2e2c","job":"test-ns-6fx77/test-trainjob-mzskk","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-6fx77"}
{"level":"Level(-2)","ts":"2026-11-02T09:38:31.872273889Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-6fx77","name":"test-trainjob-mzskk","reconcileID":"3bb0dba2-bdbf-4c5b-8dd4-cec67a9cd33e","job":"test-ns-6fx77/test-trainjob-mzskk","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-6fx77"}
{"level":"Level(-2)","ts":"2026-11-02T09:38:50.215815976Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wd7wz","name":"test-trainjob-97jgb","reconcileID":"78b1421d-c84d-4ac7-ae2a-1baa68b298dc","job":"test-ns-wd7wz/test-trainjob-97jgb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wd7wz"}
{"level":"Level(-2)","ts":"2026-11-02T09:38:50.334523613Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wd7wz","name":"test-trainjob-97jgb","reconcileID":"d0cce6a7-c45b-48b9-9d5c-22eebdd22060","job":"test-ns-wd7wz/test-trainjob-97jgb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wd7wz"}
{"level":"Level(-2)","ts":"2026-11-02T09:38:55.234569126Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wd7wz","name":"test-trainjob-97jgb","reconcileID":"74d2a7db-ec51-4033-9802-903ecb2009a2","job":"test-ns-wd7wz/test-trainjob-97jgb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wd7wz"}
{"level":"Level(-2)","ts":"2026-11-02T09:38:55.352017933Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wd7wz","name":"test-trainjob-97jgb","reconcileID":"a3bef850-cd1c-4bbf-aaae-e95b9abc1baa","job":"test-ns-wd7wz/test-trainjob-97jgb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wd7wz"}
{"level":"Level(-2)","ts":"2026-11-02T09:38:56.243906987Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wd7wz","name":"test-trainjob-97jgb","reconcileID":"0862fd75-5205-494b-be4a-eb950e2bd479","job":"test-ns-wd7wz/test-trainjob-97jgb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wd7wz"}
{"level":"Level(-2)","ts":"2026-11-02T09:38:56.362205198Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wd7wz","name":"test-trainjob-97jgb","reconcileID":"4f982869-2355-4262-bf92-9bd73e33ed52","job":"test-ns-wd7wz/test-trainjob-97jgb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wd7wz"}
{"level":"Level(-2)","ts":"2026-11-02T09:38:56.470000061Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wd7wz","name":"test-trainjob-97jgb","reconcileID":"f48160ba-0454-4b1c-8bc9-e4b889b8bfcb","job":"test-ns-wd7wz/test-trainjob-97jgb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wd7wz"}
{"level":"Level(-2)","ts":"2026-11-02T09:38:56.587171263Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wd7wz","name":"test-trainjob-97jgb","reconcileID":"dad148b7-e88c-43b6-a352-a57048e7d5d9","job":"test-ns-wd7wz/test-trainjob-97jgb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wd7wz"}
{"level":"Level(-2)","ts":"2026-11-02T09:38:56.592116116Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wd7wz","name":"test-trainjob-97jgb","reconcileID":"216cf3ac-adf9-4403-804c-6c9cfd038fcb","job":"test-ns-wd7wz/test-trainjob-97jgb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wd7wz"}
{"level":"Level(-2)","ts":"2026-11-02T09:38:56.704016121Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wd7wz","name":"test-trainjob-97jgb","reconcileID":"52e788d3-c67b-4558-891a-19eb8a0273e1","job":"test-ns-wd7wz/test-trainjob-97jgb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wd7wz"}
{"level":"Level(-2)","ts":"2026-11-02T09:41:48.058360789Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wd7wz","name":"test-trainjob-97jgb","reconcileID":"b17df554-1b65-446f-b69b-5e4fc6466c5d","job":"test-ns-wd7wz/test-trainjob-97jgb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wd7wz"}
{"level":"Level(-2)","ts":"2026-11-02T09:41:48.176813531Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wd7wz","name":"test-trainjob-97jgb","reconcileID":"8b27048a-2edc-4107-b88a-d7f2a747c9c5","job":"test-ns-wd7wz/test-trainjob-97jgb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wd7wz"}
{"level":"Level(-2)","ts":"2026-11-02T09:41:59.087375164Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wd7wz","name":"test-trainjob-97jgb","reconcileID":"15eeddbc-7dcd-4e83-9669-6dd8c5f0f3a0","job":"test-ns-wd7wz/test-trainjob-97jgb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wd7wz"}
{"level":"Level(-2)","ts":"2026-11-02T09:41:59.205891655Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wd7wz","name":"test-trainjob-97jgb","reconcileID":"f8c7e986-7b32-4f4d-a27f-66ff046d2f78","job":"test-ns-wd7wz/test-trainjob-97jgb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wd7wz"}
{"level":"Level(-2)","ts":"2026-11-02T09:42:00.100973785Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wd7wz","name":"test-trainjob-97jgb","reconcileID":"67278fc9-f347-41d9-b403-50fa9e7b5029","job":"test-ns-wd7wz/test-trainjob-97jgb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wd7wz"}
{"level":"Level(-2)","ts":"2026-11-02T09:42:00.215525727Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wd7wz","name":"test-trainjob-97jgb","reconcileID":"0ccb6271-1c57-4bf3-a5c8-5789128febee","job":"test-ns-wd7wz/test-trainjob-97jgb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wd7wz"}
{"level":"Level(-2)","ts":"2026-11-02T09:42:00.264146632Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wd7wz","name":"test-trainjob-97jgb","reconcileID":"cc03e685-3e26-4b39-80e1-39f2d02c1d23","job":"test-ns-wd7wz/test-trainjob-97jgb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wd7wz"}
{"level":"Level(-2)","ts":"2026-11-02T09:42:00.333192094Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wd7wz","name":"test-trainjob-97jgb","reconcileID":"ae6f9ba0-6f85-47a3-bfcd-f3d20ef16fb4","job":"test-ns-wd7wz/test-trainjob-97jgb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wd7wz"}
{"level":"Level(-2)","ts":"2026-11-02T09:42:00.644433236Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wd7wz","name":"test-trainjob-97jgb","reconcileID":"f430af36-6524-41e7-9a82-cd94d7315e91","job":"test-ns-wd7wz/test-trainjob-97jgb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wd7wz"}
{"level":"Level(-2)","ts":"2026-11-02T09:42:00.768148211Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wd7wz","name":"test-trainjob-97jgb","reconcileID":"5f7ff988-2cb5-40c9-bb33-de6f24e5b6f5","job":"test-ns-wd7wz/test-trainjob-97jgb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wd7wz"}
{"level":"Level(-2)","ts":"2026-11-02T09:42:12.90891458Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wd7wz","name":"test-trainjob-97jgb","reconcileID":"98e5aa86-0149-4b71-a86c-1a5196a863bf","job":"test-ns-wd7wz/test-trainjob-97jgb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wd7wz"}
{"level":"Level(-2)","ts":"2026-11-02T09:42:13.02655317Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wd7wz","name":"test-trainjob-97jgb","reconcileID":"aed15432-eb93-4c89-b4ad-1f10e49754b9","job":"test-ns-wd7wz/test-trainjob-97jgb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wd7wz"}
{"level":"Level(-2)","ts":"2026-11-02T09:42:13.919244217Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wd7wz","name":"test-trainjob-97jgb","reconcileID":"960aac5e-35e9-4a0c-8a62-c441b8d107be","job":"test-ns-wd7wz/test-trainjob-97jgb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wd7wz"}
{"level":"Level(-2)","ts":"2026-11-02T09:42:14.03697791Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wd7wz","name":"test-trainjob-97jgb","reconcileID":"7afba1ed-bd36-488d-aefd-a529452174fc","job":"test-ns-wd7wz/test-trainjob-97jgb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wd7wz"}
{"level":"Level(-2)","ts":"2026-11-02T09:42:14.928985867Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wd7wz","name":"test-trainjob-97jgb","reconcileID":"f754710e-ed68-47ac-b984-08d957ab8405","job":"test-ns-wd7wz/test-trainjob-97jgb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wd7wz"}
{"level":"Level(-2)","ts":"2026-11-02T09:42:15.047392508Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wd7wz","name":"test-trainjob-97jgb","reconcileID":"2e7bfd1a-489a-40ff-99d2-dab4e7832819","job":"test-ns-wd7wz/test-trainjob-97jgb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wd7wz"}
{"level":"Level(-2)","ts":"2026-11-02T09:42:15.088112637Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wd7wz","name":"test-trainjob-97jgb","reconcileID":"562dac9a-f2cf-4a71-b4b7-c7aaa326ceff","job":"test-ns-wd7wz/test-trainjob-97jgb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wd7wz"}
{"level":"Level(-2)","ts":"2026-11-02T09:42:15.16536624Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wd7wz","name":"test-trainjob-97jgb","reconcileID":"8984b622-ff9d-4903-b5f6-c6073b9dd3d3","job":"test-ns-wd7wz/test-trainjob-97jgb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wd7wz"}
{"level":"Level(-2)","ts":"2026-11-02T09:42:21.362079056Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-wd7wz","name":"test-trainjob-97jgb","reconcileID":"687db25d-0c9a-4fce-899c-28cb5ec7b6e6","job":"test-ns-wd7wz/test-trainjob-97jgb","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-wd7wz"}
{"level":"Level(-2)","ts":"2026-11-02T09:44:14.409840396Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-65tqz","name":"g0fa95fc73e0","reconcileID":"41d8c4fa-42ed-4fa5-9912-65fb804ec100","job":"test-ns-65tqz/g0fa95fc73e0","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-65tqz"}
{"level":"Level(-2)","ts":"2026-11-02T09:44:14.889009599Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-65tqz","name":"g0fa95fc73e0","reconcileID":"ddcc5cbc-a1e7-4f88-9f43-aceb0aedc153","job":"test-ns-65tqz/g0fa95fc73e0","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-65tqz"}
{"level":"Level(-2)","ts":"2026-11-02T09:44:15.11550307Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-65tqz","name":"g0fa95fc73e0","reconcileID":"e45d6a77-7a3a-4370-8110-703d38a7642c","job":"test-ns-65tqz/g0fa95fc73e0","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-65tqz"}
{"level":"Level(-2)","ts":"2026-11-02T09:44:15.233655515Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-65tqz","name":"g0fa95fc73e0","reconcileID":"fd75f41a-633f-4009-aba1-e36771264d9a","job":"test-ns-65tqz/g0fa95fc73e0","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-65tqz"}
{"level":"Level(-2)","ts":"2026-11-02T09:44:15.6239555Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-65tqz","name":"g0fa95fc73e0","reconcileID":"eaa70d09-6bd8-4376-9eac-b326e226838a","job":"test-ns-65tqz/g0fa95fc73e0","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-65tqz"}
{"level":"Level(-2)","ts":"2026-11-02T09:44:15.741575128Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-65tqz","name":"g0fa95fc73e0","reconcileID":"64b07693-d1bb-4349-ae6b-7c8dd779ed66","job":"test-ns-65tqz/g0fa95fc73e0","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-65tqz"}
{"level":"Level(-2)","ts":"2026-11-02T09:44:17.945030914Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-65tqz","name":"g0fa95fc73e0","reconcileID":"b972c95b-f486-4627-992c-025e21fb71b2","job":"test-ns-65tqz/g0fa95fc73e0","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-65tqz"}
{"level":"Level(-2)","ts":"2026-11-02T09:44:18.065470067Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-65tqz","name":"g0fa95fc73e0","reconcileID":"175e2515-bdf3-4ec7-a1ed-a9c1dbd80e43","job":"test-ns-65tqz/g0fa95fc73e0","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-65tqz"}
{"level":"Level(-2)","ts":"2026-11-02T09:44:51.063416556Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-65tqz","name":"g0fa95fc73e0","reconcileID":"cdcca0aa-22eb-4c8f-89a0-9dadfdbcb212","job":"test-ns-65tqz/g0fa95fc73e0","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-65tqz"}
{"level":"Level(-2)","ts":"2026-11-02T09:44:51.182663974Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-65tqz","name":"g0fa95fc73e0","reconcileID":"76461926-b7b4-4209-beab-bf7a73e79abf","job":"test-ns-65tqz/g0fa95fc73e0","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-65tqz"}
{"level":"Level(-2)","ts":"2026-11-02T09:44:52.078635484Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-65tqz","name":"g0fa95fc73e0","reconcileID":"82d0eca9-7897-4a1e-8661-3ca03310f83c","job":"test-ns-65tqz/g0fa95fc73e0","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-65tqz"}
{"level":"Level(-2)","ts":"2026-11-02T09:44:52.193157201Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-65tqz","name":"g0fa95fc73e0","reconcileID":"d9ec07bf-d2ac-4976-9416-ab0c19bdafc2","job":"test-ns-65tqz/g0fa95fc73e0","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-65tqz"}
{"level":"Level(-2)","ts":"2026-11-02T09:44:52.231256305Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-65tqz","name":"g0fa95fc73e0","reconcileID":"bc80a7d9-dba3-48e5-a782-d46465c73ff3","job":"test-ns-65tqz/g0fa95fc73e0","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-65tqz"}
{"level":"Level(-2)","ts":"2026-11-02T09:44:52.313078692Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-65tqz","name":"g0fa95fc73e0","reconcileID":"4af839a5-b9b7-441c-b4be-f826a3ab8456","job":"test-ns-65tqz/g0fa95fc73e0","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-65tqz"}
{"level":"Level(-2)","ts":"2026-11-02T09:44:56.974430953Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:141","msg":"ResourceFlavor create event","replica-role":"leader","resourceFlavor":{"name":"rf-f828t"}}
{"level":"Level(-2)","ts":"2026-11-02T09:44:56.974589453Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-f828t","reconcileID":"5c9f2670-5d14-47df-ba0a-ace892ee17ac"}
{"level":"Level(-2)","ts":"2026-11-02T09:44:57.341422785Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:351","msg":"ClusterQueue create event","replica-role":"leader","clusterQueue":{"name":"cq-z9f8j"}}
{"level":"Level(-2)","ts":"2026-11-02T09:44:57.341693024Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-z9f8j","reconcileID":"3d1eb337-b450-41e2-bc56-4c1acad441d6"}
{"level":"Level(-2)","ts":"2026-11-02T09:44:57.361092771Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:240","msg":"LocalQueue create event","replica-role":"leader","localQueue":{"name":"lq-bvgjr","namespace":"test-ns-wdbkl"}}
{"level":"Level(-2)","ts":"2026-11-02T09:44:57.361294686Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-wdbkl","name":"lq-bvgjr","reconcileID":"948804a4-a744-40c4-b330-244a1845d5af"}
{"level":"Level(-2)","ts":"2026-11-02T09:44:57.398707568Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-z9f8j"}}
{"level":"Level(-2)","ts":"2026-11-02T09:44:57.398895614Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-z9f8j","reconcileID":"1c712614-270f-4d61-99ac-8fa744de74c7"}
{"level":"Level(-2)","ts":"2026-11-02T09:44:57.41980264Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-wdbkl","name":"lq-bvgjr","reconcileID":"b140a9ea-4805-4862-86cd-45dae397daa4"}
{"level":"Level(-2)","ts":"2026-11-02T09:44:57.420060839Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-bvgjr","namespace":"test-ns-wdbkl"}}
{"level":"Level(-3)","ts":"2026-11-02T09:44:57.477186196Z","logger":"localqueue-reconciler","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-wdbkl","name":"lq-bvgjr","reconcileID":"b140a9ea-4805-4862-86cd-45dae397daa4","error":"Operation cannot be fulfilled on localqueues.kueue.x-k8s.io \"lq-bvgjr\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-11-02T09:44:57.477306312Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-wdbkl","name":"lq-bvgjr","reconcileID":"5715cb6e-f0e9-4550-a9e2-debf120948ed"}
{"level":"Level(-2)","ts":"2026-11-02T09:44:57.53568517Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-bvgjr","namespace":"test-ns-wdbkl"}}
{"level":"Level(-2)","ts":"2026-11-02T09:44:57.535809536Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-wdbkl","name":"lq-bvgjr","reconcileID":"23a51e52-3f98-4371-87dd-e984885e2cec"}
{"level":"Level(-2)","ts":"2026-11-02T09:44:58.262866035Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-wdbkl","name":"dw-deployment-7fjs6-86bddc58fc-h2x2t","reconcileID":"86123ccc-8c88-4d56-a94b-f5d32b733175","job":"test-ns-wdbkl/dw-deployment-7fjs6-86bddc58fc-h2x2t","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-02T09:44:58.316641321Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1.PriorityClass","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"debug","ts":"2026-11-02T09:44:58.830966279Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"dw-deployment-7fjs6-86bddc58fc-h2x2t","namespace":"test-ns-wdbkl"},"kind":"Pod","apiVersion":"v1","type":"Normal","reason":"CreatedWorkload","action":"CreatedWorkload","note":"Created Workload: test-ns-wdbkl/pod-dw-deployment-7fjs6-86bddc58fc-h2x2t-85c32"}
{"level":"Level(-2)","ts":"2026-11-02T09:44:58.831641579Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1276","msg":"Workload create event","replica-role":"leader","workload":{"name":"pod-dw-deployment-7fjs6-86bddc58fc-h2x2t-85c32","namespace":"test-ns-wdbkl"},"queue":"lq-bvgjr","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-02T09:44:58.831721942Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-wdbkl","name":"pod-dw-deployment-7fjs6-86bddc58fc-h2x2t-85c32","reconcileID":"4dad1e67-ebd0-4e8b-8751-ce50331c1351"}
{"level":"Level(-2)","ts":"2026-11-02T09:44:58.831740087Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-wdbkl","name":"dw-deployment-7fjs6-86bddc58fc-h2x2t","reconcileID":"ea411c49-4ade-4998-a02b-52dd2491ccdb","job":"test-ns-wdbkl/dw-deployment-7fjs6-86bddc58fc-h2x2t","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-02T09:44:58.831776073Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-wdbkl","name":"pod-dw-deployment-7fjs6-86bddc58fc-h2x2t-85c32","reconcileID":"4dad1e67-ebd0-4e8b-8751-ce50331c1351","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T09:44:58.831904552Z","logger":"scheduler","caller":"scheduler/scheduler.go:308","msg":"Obtained heads","replica-role":"follower","schedulingCycle":1,"headCount":1,"waitDuration":2035.636881203}
{"level":"Level(-2)","ts":"2026-11-02T09:44:58.831950061Z","logger":"scheduler","caller":"scheduler/scheduler.go:323","msg":"Snapshot taken","replica-role":"follower","schedulingCycle":1,"duration":0.000015946}
{"level":"Level(-2)","ts":"2026-11-02T09:44:58.83199867Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-wdbkl","name":"pod-dw-deployment-7fjs6-86bddc58fc-h2x2t-85c32","reconcileID":"0bb3a23f-2d8f-475b-8b9f-9c90aaaace72"}
{"level":"Level(-2)","ts":"2026-11-02T09:44:58.833166819Z","logger":"scheduler","caller":"scheduler/scheduler.go:328","msg":"Nomination done","replica-role":"follower","schedulingCycle":1,"entries":1,"inadmissibleEntries":0,"duration":0.001200145}
{"level":"Level(-2)","ts":"2026-11-02T09:44:58.833194781Z","logger":"scheduler","caller":"scheduler/scheduler.go:406","msg":"Attempting to schedule workload","replica-role":"follower","schedulingCycle":1,"workload":{"name":"pod-dw-deployment-7fjs6-86bddc58fc-h2x2t-85c32","namespace":"test-ns-wdbkl"},"clusterQueue":{"name":"cq-z9f8j"}}
{"level":"Level(-2)","ts":"2026-11-02T09:44:58.833332437Z","logger":"scheduler","caller":"scheduler/scheduler.go:1041","msg":"Workload assumed in the cache","replica-role":"follower","schedulingCycle":1,"workload":{"name":"pod-dw-deployment-7fjs6-86bddc58fc-h2x2t-85c32","namespace":"test-ns-wdbkl"},"clusterQueue":{"name":"cq-z9f8j"}}
{"level":"Level(-2)","ts":"2026-11-02T09:44:58.835023434Z","logger":"scheduler","caller":"scheduler/scheduler.go:362","msg":"Workload processing done","replica-role":"follower","schedulingCycle":1,"duration":0.001833876}
{"level":"Level(-2)","ts":"2026-11-02T09:44:58.835088062Z","logger":"scheduler","caller":"scheduler/scheduler.go:297","msg":"Scheduling cycle complete","replica-role":"follower","schedulingCycle":1,"duration":2035.640064604}
{"level":"Level(-2)","ts":"2026-11-02T09:44:58.835124527Z","logger":"scheduler","caller":"scheduler/scheduler.go:295","msg":"Scheduling cycle starts","replica-role":"leader","schedulingCycle":2}
{"level":"Level(-2)","ts":"2026-11-02T09:44:58.955837775Z","logger":"scheduler","caller":"scheduler/scheduler.go:1000","msg":"Workload successfully admitted and assigned flavors","replica-role":"follower","schedulingCycle":1,"workload":{"name":"pod-dw-deployment-7fjs6-86bddc58fc-h2x2t-85c32","namespace":"test-ns-wdbkl"},"clusterQueue":{"name":"cq-z9f8j"},"assignments":[{"name":"main","flavors":{"cpu":"rf-f828t","memory":"rf-f828t"},"resourceUsage":{"cpu":"1","memory":"3Gi"},"count":1}]}
{"level":"debug","ts":"2026-11-02T09:44:58.955881211Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"pod-dw-deployment-7fjs6-86bddc58fc-h2x2t-85c32","namespace":"test-ns-wdbkl"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"QuotaReserved","action":"QuotaReserved","note":"Quota reserved in ClusterQueue cq-z9f8j, wait time since queued was 1s"}
{"level":"debug","ts":"2026-11-02T09:44:58.955925978Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"pod-dw-deployment-7fjs6-86bddc58fc-h2x2t-85c32","namespace":"test-ns-wdbkl"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"Admitted","action":"Admitted","note":"Admitted by ClusterQueue cq-z9f8j, wait time since reservation was 1s"}
{"level":"Level(-2)","ts":"2026-11-02T09:44:58.955920915Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-wdbkl","name":"pod-dw-deployment-7fjs6-86bddc58fc-h2x2t-85c32","reconcileID":"14cde60d-68a8-45c8-ab23-0107aa88e3b9"}
{"level":"Level(-2)","ts":"2026-11-02T09:44:58.955950991Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"pod-dw-deployment-7fjs6-86bddc58fc-h2x2t-85c32","namespace":"test-ns-wdbkl"},"queue":"lq-bvgjr","status":"admitted","prevStatus":"pending","clusterQueue":"cq-z9f8j"}
{"level":"Level(-2)","ts":"2026-11-02T09:44:58.956008749Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-wdbkl","name":"pod-dw-deployment-7fjs6-86bddc58fc-h2x2t-85c32","reconcileID":"14cde60d-68a8-45c8-ab23-0107aa88e3b9","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T09:44:58.956137957Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-wdbkl","name":"dw-deployment-7fjs6-86bddc58fc-h2x2t","reconcileID":"8321bc69-5021-4e5a-9744-8bfc6ae1b9f9","job":"test-ns-wdbkl/dw-deployment-7fjs6-86bddc58fc-h2x2t","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-02T09:44:58.956275949Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-wdbkl","name":"pod-dw-deployment-7fjs6-86bddc58fc-h2x2t-85c32","reconcileID":"0c8042cb-7d7e-4932-8717-e8c344d6d3f6"}
{"level":"Level(-2)","ts":"2026-11-02T09:44:58.956411917Z","logger":"v1_pod","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-wdbkl","name":"dw-deployment-7fjs6-86bddc58fc-h2x2t","reconcileID":"8321bc69-5021-4e5a-9744-8bfc6ae1b9f9","job":"test-ns-wdbkl/dw-deployment-7fjs6-86bddc58fc-h2x2t","gvk":"/v1, Kind=Pod"}
{"level":"debug","ts":"2026-11-02T09:44:59.077714936Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"dw-deployment-7fjs6-86bddc58fc-h2x2t","namespace":"test-ns-wdbkl"},"kind":"Pod","apiVersion":"v1","type":"Normal","reason":"Started","action":"Started","note":"Admitted by clusterQueue cq-z9f8j"}
{"level":"Level(-2)","ts":"2026-11-02T09:44:59.078874651Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-wdbkl","name":"dw-deployment-7fjs6-86bddc58fc-h2x2t","reconcileID":"2d5a1a18-0c10-4467-9afa-8f4856c78bcc","job":"test-ns-wdbkl/dw-deployment-7fjs6-86bddc58fc-h2x2t","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-02T09:44:59.089938552Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-wdbkl","name":"dw-deployment-7fjs6-86bddc58fc-h2x2t","reconcileID":"c1e6372b-058a-4eb0-b602-91cac5af62ff","job":"test-ns-wdbkl/dw-deployment-7fjs6-86bddc58fc-h2x2t","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-02T09:44:59.154521218Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-wdbkl","name":"dw-deployment-7fjs6-86bddc58fc-h2x2t","reconcileID":"9945f401-11db-443f-b355-21d01f5a8cc5","job":"test-ns-wdbkl/dw-deployment-7fjs6-86bddc58fc-h2x2t","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-02T09:44:59.210147363Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-wdbkl","name":"dw-deployment-7fjs6-86bddc58fc-h2x2t","reconcileID":"8b440994-160e-4a86-9f9d-29ba3043cb20","job":"test-ns-wdbkl/dw-deployment-7fjs6-86bddc58fc-h2x2t","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-02T09:44:59.832825204Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-z9f8j","reconcileID":"c31ef289-b055-4d49-a12c-190714dab9d8"}
{"level":"Level(-2)","ts":"2026-11-02T09:44:59.832813493Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-wdbkl","name":"lq-bvgjr","reconcileID":"3c108fa1-c614-4c7e-b4d5-c807f52e9794"}
{"level":"Level(-2)","ts":"2026-11-02T09:44:59.891108364Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-bvgjr","namespace":"test-ns-wdbkl"}}
{"level":"Level(-2)","ts":"2026-11-02T09:44:59.891219502Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-wdbkl","name":"lq-bvgjr","reconcileID":"d8b50dbf-2973-42dd-9860-dc68f0373088"}
{"level":"Level(-2)","ts":"2026-11-02T09:44:59.894065937Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-z9f8j"}}
{"level":"Level(-2)","ts":"2026-11-02T09:44:59.895183409Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-z9f8j","reconcileID":"d698deb9-8fa3-4e9c-a6f1-6e11369200f8"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:00.080109446Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-wdbkl","name":"pod-dw-deployment-7fjs6-86bddc58fc-h2x2t-85c32","reconcileID":"40592746-cb7e-43fa-a3f2-a42876b7dd57"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:00.08103543Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-wdbkl","name":"dw-deployment-7fjs6-86bddc58fc-h2x2t","reconcileID":"831e11d9-50d9-438f-bc8f-5dc1da381f8c","job":"test-ns-wdbkl/dw-deployment-7fjs6-86bddc58fc-h2x2t","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:01.081337201Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-wdbkl","name":"pod-dw-deployment-7fjs6-86bddc58fc-h2x2t-85c32","reconcileID":"492d2a36-b2e1-4439-ab8c-ef633565366e"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:02.006393604Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-wdbkl","name":"dw-deployment-7fjs6-86bddc58fc-h2x2t","reconcileID":"37890b74-201d-4168-9c53-e6a1fe08ecbc","job":"test-ns-wdbkl/dw-deployment-7fjs6-86bddc58fc-h2x2t","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:03.007773505Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-wdbkl","name":"pod-dw-deployment-7fjs6-86bddc58fc-h2x2t-85c32","reconcileID":"46d6097c-c64f-41a5-8bf0-ba1572284e70"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:13.340019044Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-wdbkl","name":"d6e308ebb23a","reconcileID":"c061c9e0-a3b9-4947-bd52-8d58caae20f5","job":"test-ns-wdbkl/d6e308ebb23a","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:13.671106204Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-wdbkl","name":"d6e308ebb23a","reconcileID":"f5889a1a-4bbc-442a-ab4c-608cd0ef1bb1","job":"test-ns-wdbkl/d6e308ebb23a","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"debug","ts":"2026-11-02T09:45:13.671073895Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"d6e308ebb23a","namespace":"test-ns-wdbkl"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"CreatedWorkload","action":"CreatedWorkload","note":"Created Workload: test-ns-wdbkl/trainjob-d6e308ebb23a-1878b"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:13.671505992Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-wdbkl","name":"trainjob-d6e308ebb23a-1878b","reconcileID":"da29a133-165f-41bd-8dcf-7a92835b2c00"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:13.671534012Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1276","msg":"Workload create event","replica-role":"leader","workload":{"name":"trainjob-d6e308ebb23a-1878b","namespace":"test-ns-wdbkl"},"queue":"lq-bvgjr","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:13.671562955Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-wdbkl","name":"trainjob-d6e308ebb23a-1878b","reconcileID":"da29a133-165f-41bd-8dcf-7a92835b2c00","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T09:45:13.671731376Z","logger":"scheduler","caller":"scheduler/scheduler.go:308","msg":"Obtained heads","replica-role":"leader","schedulingCycle":2,"headCount":1,"waitDuration":14.836606192}
{"level":"Level(-2)","ts":"2026-11-02T09:45:13.671771703Z","logger":"scheduler","caller":"scheduler/scheduler.go:323","msg":"Snapshot taken","replica-role":"leader","schedulingCycle":2,"duration":0.000014207}
{"level":"Level(-2)","ts":"2026-11-02T09:45:13.671794507Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-wdbkl","name":"trainjob-d6e308ebb23a-1878b","reconcileID":"c4414118-8b1f-42e8-ba35-cd2042382a6c"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:13.671858859Z","logger":"scheduler","caller":"scheduler/scheduler.go:328","msg":"Nomination done","replica-role":"leader","schedulingCycle":2,"entries":1,"inadmissibleEntries":0,"duration":0.000079413}
{"level":"Level(-2)","ts":"2026-11-02T09:45:13.671878549Z","logger":"scheduler","caller":"scheduler/scheduler.go:406","msg":"Attempting to schedule workload","replica-role":"leader","schedulingCycle":2,"workload":{"name":"trainjob-d6e308ebb23a-1878b","namespace":"test-ns-wdbkl"},"clusterQueue":{"name":"cq-z9f8j"}}
{"level":"Level(-2)","ts":"2026-11-02T09:45:13.671973559Z","logger":"scheduler","caller":"scheduler/scheduler.go:1041","msg":"Workload assumed in the cache","replica-role":"leader","schedulingCycle":2,"workload":{"name":"trainjob-d6e308ebb23a-1878b","namespace":"test-ns-wdbkl"},"clusterQueue":{"name":"cq-z9f8j"}}
{"level":"Level(-2)","ts":"2026-11-02T09:45:13.672008615Z","logger":"scheduler","caller":"scheduler/scheduler.go:362","msg":"Workload processing done","replica-role":"leader","schedulingCycle":2,"duration":0.000133677}
{"level":"Level(-2)","ts":"2026-11-02T09:45:13.672024714Z","logger":"scheduler","caller":"scheduler/scheduler.go:297","msg":"Scheduling cycle complete","replica-role":"leader","schedulingCycle":2,"duration":14.836899989}
{"level":"Level(-2)","ts":"2026-11-02T09:45:13.672061884Z","logger":"scheduler","caller":"scheduler/scheduler.go:295","msg":"Scheduling cycle starts","replica-role":"leader","schedulingCycle":3}
{"level":"Level(-2)","ts":"2026-11-02T09:45:13.78552626Z","logger":"scheduler","caller":"scheduler/scheduler.go:1000","msg":"Workload successfully admitted and assigned flavors","replica-role":"leader","schedulingCycle":2,"workload":{"name":"trainjob-d6e308ebb23a-1878b","namespace":"test-ns-wdbkl"},"clusterQueue":{"name":"cq-z9f8j"},"assignments":[{"name":"node","flavors":{"cpu":"rf-f828t","memory":"rf-f828t"},"resourceUsage":{"cpu":"4","memory":"16Gi"},"count":2}]}
{"level":"debug","ts":"2026-11-02T09:45:13.785582078Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-d6e308ebb23a-1878b","namespace":"test-ns-wdbkl"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"Admitted","action":"Admitted","note":"Admitted by ClusterQueue cq-z9f8j, wait time since reservation was 1s"}
{"level":"debug","ts":"2026-11-02T09:45:13.785596884Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-d6e308ebb23a-1878b","namespace":"test-ns-wdbkl"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"QuotaReserved","action":"QuotaReserved","note":"Quota reserved in ClusterQueue cq-z9f8j, wait time since queued was 1s"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:13.785810197Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-d6e308ebb23a-1878b","namespace":"test-ns-wdbkl"},"queue":"lq-bvgjr","status":"admitted","prevStatus":"pending","clusterQueue":"cq-z9f8j"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:13.785814946Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-wdbkl","name":"trainjob-d6e308ebb23a-1878b","reconcileID":"523a320b-9565-4e54-a2df-dbabef9a8eed"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:13.785870281Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-wdbkl","name":"trainjob-d6e308ebb23a-1878b","reconcileID":"523a320b-9565-4e54-a2df-dbabef9a8eed","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T09:45:13.786089533Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-wdbkl","name":"trainjob-d6e308ebb23a-1878b","reconcileID":"94de1a9d-9323-4a0d-ba1d-03a3f4c03974"}
{"level":"error","ts":"2026-11-02T09:45:14.015720182Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-wdbkl","name":"d6e308ebb23a","reconcileID":"f5889a1a-4bbc-442a-ab4c-608cd0ef1bb1","error":"workloads.kueue.x-k8s.io \"trainjob-d6e308ebb23a-1878b\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:14.01585031Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-wdbkl","name":"d6e308ebb23a","reconcileID":"67e01e3a-3282-448d-8ab2-44f384ffbe5f","job":"test-ns-wdbkl/d6e308ebb23a","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:14.177856284Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-wdbkl","name":"d6e308ebb23a","reconcileID":"67e01e3a-3282-448d-8ab2-44f384ffbe5f","job":"test-ns-wdbkl/d6e308ebb23a","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:14.52606565Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-wdbkl","name":"d6e308ebb23a","reconcileID":"cdc44b91-6113-4f0e-aeef-00ac4c3b5a3b","job":"test-ns-wdbkl/d6e308ebb23a","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"debug","ts":"2026-11-02T09:45:14.526064114Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"d6e308ebb23a","namespace":"test-ns-wdbkl"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"Started","action":"Started","note":"Admitted by clusterQueue cq-z9f8j"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:14.672566112Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-wdbkl","name":"lq-bvgjr","reconcileID":"7ffb86de-7054-43b2-bebe-cf7ee1056ee8"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:14.672592245Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-z9f8j","reconcileID":"4c8396e0-caa5-4ecc-bc9c-f39d0d52f562"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:14.687181344Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-wdbkl","name":"d6e308ebb23a","reconcileID":"cdc44b91-6113-4f0e-aeef-00ac4c3b5a3b","job":"test-ns-wdbkl/d6e308ebb23a","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:14.729671181Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-bvgjr","namespace":"test-ns-wdbkl"}}
{"level":"Level(-2)","ts":"2026-11-02T09:45:14.729808812Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-wdbkl","name":"lq-bvgjr","reconcileID":"d5f6e185-0220-4090-b319-a669b4d9a0e7"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:14.73240488Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-z9f8j"}}
{"level":"Level(-2)","ts":"2026-11-02T09:45:14.73260781Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-z9f8j","reconcileID":"4b60854a-fd80-4e54-b1de-0bd979b6961f"}
{"level":"Level(-3)","ts":"2026-11-02T09:45:14.797341615Z","logger":"trainjob","caller":"jobframework/reconciler.go:594","msg":"Unsuspending job","replica-role":"leader","namespace":"test-ns-wdbkl","name":"d6e308ebb23a","reconcileID":"cdc44b91-6113-4f0e-aeef-00ac4c3b5a3b","job":"test-ns-wdbkl/d6e308ebb23a","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"d6e308ebb23a\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-3)","ts":"2026-11-02T09:45:14.797428226Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-wdbkl","name":"d6e308ebb23a","reconcileID":"cdc44b91-6113-4f0e-aeef-00ac4c3b5a3b","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"d6e308ebb23a\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:14.79755035Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-wdbkl","name":"d6e308ebb23a","reconcileID":"418a19b0-ca9d-4719-9dd2-b6efe6a91754","job":"test-ns-wdbkl/d6e308ebb23a","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:14.928625636Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1.PartialObjectMetadata","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:14.959957699Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-wdbkl","name":"d6e308ebb23a","reconcileID":"702489f9-8523-47cb-a2cf-f33863c063e0","job":"test-ns-wdbkl/d6e308ebb23a","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:15.032200691Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1.PartialObjectMetadata","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:15.121467449Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-wdbkl","name":"d6e308ebb23a","reconcileID":"17883f1b-ed6f-4969-8944-f8f9be764302","job":"test-ns-wdbkl/d6e308ebb23a","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:15.282818409Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-wdbkl","name":"d6e308ebb23a","reconcileID":"896cffa7-28e3-4cda-99b9-1fd2519d5198","job":"test-ns-wdbkl/d6e308ebb23a","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:15.448753137Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-wdbkl","name":"d6e308ebb23a","reconcileID":"ff189d95-0a8a-4e29-be93-35d0a31859f8","job":"test-ns-wdbkl/d6e308ebb23a","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:15.609751733Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-wdbkl","name":"d6e308ebb23a","reconcileID":"57fb2a0f-3fb1-49c1-bd99-a1398f189582","job":"test-ns-wdbkl/d6e308ebb23a","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:16.194183267Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-wdbkl","name":"trainjob-d6e308ebb23a-1878b","reconcileID":"d8af1743-cc47-44fd-83e2-5f80bea08f0e"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:17.284423387Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-wdbkl","name":"trainjob-d6e308ebb23a-1878b","reconcileID":"474cde28-554b-4ad6-9209-4a8d2ff8bf11"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:17.355969819Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-wdbkl","name":"d6e308ebb23a","reconcileID":"5dc2db8c-afe1-483a-8ce7-50df9913d469","job":"test-ns-wdbkl/d6e308ebb23a","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:17.518110275Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-wdbkl","name":"d6e308ebb23a","reconcileID":"31f3cfb7-08f4-4cdf-9c5b-d7b8a98a4277","job":"test-ns-wdbkl/d6e308ebb23a","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:50.402722076Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-wdbkl","name":"trainjob-d6e308ebb23a-1878b","reconcileID":"b71cc4eb-3d40-4ab0-897d-f2c2ba77bb63"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:50.434465652Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-wdbkl","name":"d6e308ebb23a","reconcileID":"7423cb4c-8fd3-4cba-a1a8-32d8c5133ac6","job":"test-ns-wdbkl/d6e308ebb23a","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:50.596284365Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-wdbkl","name":"d6e308ebb23a","reconcileID":"4cbc9a58-2f4a-4a93-a6e9-70cb4beeb861","job":"test-ns-wdbkl/d6e308ebb23a","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:51.438121327Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-wdbkl","name":"d6e308ebb23a","reconcileID":"bec067ac-e0e4-4eae-87f6-a8ba90ff4b4d","job":"test-ns-wdbkl/d6e308ebb23a","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:51.484127621Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-wdbkl","name":"trainjob-d6e308ebb23a-1878b","reconcileID":"ebddf51c-e760-4197-aab7-59af1ae81f81"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:51.599510283Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-wdbkl","name":"d6e308ebb23a","reconcileID":"27c0367d-1162-4ec9-8d49-641e65b19b7b","job":"test-ns-wdbkl/d6e308ebb23a","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:51.760697834Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-wdbkl","name":"d6e308ebb23a","reconcileID":"22fcda43-e886-4b46-b923-ba929f6827f1","job":"test-ns-wdbkl/d6e308ebb23a","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:52.043512561Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-d6e308ebb23a-1878b","namespace":"test-ns-wdbkl"},"queue":"lq-bvgjr","status":"finished","prevStatus":"admitted","clusterQueue":"cq-z9f8j"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:52.04357404Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-wdbkl","name":"trainjob-d6e308ebb23a-1878b","reconcileID":"9efb48d0-6a08-446d-8145-23286f0c9acd"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:52.043633402Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-wdbkl","name":"trainjob-d6e308ebb23a-1878b","reconcileID":"9efb48d0-6a08-446d-8145-23286f0c9acd","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T09:45:52.043885645Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-wdbkl","name":"trainjob-d6e308ebb23a-1878b","reconcileID":"4429d68e-299f-42d2-a70c-f8b2af9ca46f"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:52.044841751Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-wdbkl","name":"d6e308ebb23a","reconcileID":"ebcd72f8-a8cb-4aa2-9f98-14fe8b871915","job":"test-ns-wdbkl/d6e308ebb23a","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"debug","ts":"2026-11-02T09:45:52.044848161Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"d6e308ebb23a","namespace":"test-ns-wdbkl"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"FinishedWorkload","action":"FinishedWorkload","note":"Workload 'test-ns-wdbkl/trainjob-d6e308ebb23a-1878b' is declared finished"}
{"level":"debug","ts":"2026-11-02T09:45:52.205499689Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"d6e308ebb23a","namespace":"test-ns-wdbkl"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"FinishedWorkload","action":"FinishedWorkload","note":"Workload 'test-ns-wdbkl/trainjob-d6e308ebb23a-1878b' is declared finished"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:52.321447157Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-d6e308ebb23a-1878b","namespace":"test-ns-wdbkl"},"queue":"lq-bvgjr","status":"finished","clusterQueue":"cq-z9f8j"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:52.32180193Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-wdbkl","name":"trainjob-d6e308ebb23a-1878b","reconcileID":"d74e6384-4072-4cc6-be46-bd8e62991d2f"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:52.322195637Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-wdbkl","name":"d6e308ebb23a","reconcileID":"c34e4cce-6419-4a92-8d58-ae5070ca1d6c","job":"test-ns-wdbkl/d6e308ebb23a","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:52.323966509Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-wdbkl","name":"trainjob-d6e308ebb23a-1878b","reconcileID":"8f3474e0-7fdf-49a1-b3cc-08f432a4267a"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:52.324101629Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-wdbkl","name":"trainjob-d6e308ebb23a-1878b","reconcileID":"8f3474e0-7fdf-49a1-b3cc-08f432a4267a","isDeleted":false}
{"level":"debug","ts":"2026-11-02T09:45:52.485823584Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"d6e308ebb23a","namespace":"test-ns-wdbkl"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"FinishedWorkload","action":"FinishedWorkload","note":"Workload 'test-ns-wdbkl/trainjob-d6e308ebb23a-1878b' is declared finished"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:52.487564928Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-wdbkl","name":"trainjob-d6e308ebb23a-1878b","reconcileID":"4f95b88c-3a9a-4530-9d13-54c12c6fcd27"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:53.044906173Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-wdbkl","name":"lq-bvgjr","reconcileID":"abaa783e-4e2a-4175-a8b7-4a97a2a0e2fa"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:53.044907315Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-z9f8j","reconcileID":"3d0a2f1a-e698-4143-ba3d-1408f52bfd7b"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:53.104238247Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-bvgjr","namespace":"test-ns-wdbkl"}}
{"level":"Level(-2)","ts":"2026-11-02T09:45:53.10455568Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-wdbkl","name":"lq-bvgjr","reconcileID":"646594dd-de0a-4c5c-95ea-07d306a3dce0"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:53.105308774Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-z9f8j"}}
{"level":"Level(-2)","ts":"2026-11-02T09:45:53.105460244Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-z9f8j","reconcileID":"23feca81-0941-4da6-859d-a05b303b7376"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:53.861178947Z","logger":"trainjob","caller":"workload/workload.go:1810","msg":"Workload is orphaned; finishing to release quota","replica-role":"leader","namespace":"test-ns-wdbkl","name":"d6e308ebb23a","reconcileID":"3d1e95ca-f963-48e4-96c7-d3e2f12fd9a9","job":"test-ns-wdbkl/d6e308ebb23a","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:53.910589803Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1312","msg":"Workload delete event","replica-role":"leader","workload":{"name":"trainjob-d6e308ebb23a-1878b","namespace":"test-ns-wdbkl"},"queue":"lq-bvgjr","status":"finished"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:53.910602021Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-wdbkl","name":"trainjob-d6e308ebb23a-1878b","reconcileID":"ea762ddb-73ea-49c8-a896-249b6d42413b"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:53.910867025Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-wdbkl","name":"trainjob-d6e308ebb23a-1878b","reconcileID":"ea762ddb-73ea-49c8-a896-249b6d42413b","isDeleted":true}
{"level":"Level(-2)","ts":"2026-11-02T09:45:54.878421382Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-z9f8j"}}
{"level":"Level(-2)","ts":"2026-11-02T09:45:54.878520774Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-z9f8j","reconcileID":"fd489317-e2da-4d0e-a6e3-611c0ab6884c"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:54.878638794Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-wdbkl","name":"pod-dw-deployment-7fjs6-86bddc58fc-h2x2t-85c32","reconcileID":"30cf1751-fb46-43a0-9779-c665d7f1e422"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:54.896952586Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:171","msg":"ResourceFlavor update event","replica-role":"leader","resourceFlavor":{"name":"rf-f828t"}}
{"level":"Level(-2)","ts":"2026-11-02T09:45:54.897120862Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-f828t","reconcileID":"51a65e1e-7a66-4280-8023-c3c48cdf06ed"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:54.910879157Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-wdbkl","name":"lq-bvgjr","reconcileID":"360d345e-21d8-4022-8ba1-acf66f62873b"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:54.938933411Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-z9f8j"}}
{"level":"Level(-2)","ts":"2026-11-02T09:45:54.938997696Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-wdbkl","name":"lq-bvgjr","reconcileID":"03488aee-01b2-4728-8f76-efad1819852e"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:54.939020568Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-z9f8j","reconcileID":"b3b6454f-a477-490d-9c2c-6ca0b4b3c19e"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:54.939183916Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-wdbkl","name":"pod-dw-deployment-7fjs6-86bddc58fc-h2x2t-85c32","reconcileID":"1fa3a504-a82b-4c0a-b940-18b2a4db93bb"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:54.998162709Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-bvgjr","namespace":"test-ns-wdbkl"}}
{"level":"Level(-2)","ts":"2026-11-02T09:45:54.999465597Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-wdbkl","name":"lq-bvgjr","reconcileID":"2887a86a-e5b4-4d2d-ab87-5db48d2b653a"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:55.310516717Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:141","msg":"ResourceFlavor create event","replica-role":"leader","resourceFlavor":{"name":"rf-fm4cn"}}
{"level":"Level(-2)","ts":"2026-11-02T09:45:55.310625927Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-fm4cn","reconcileID":"7884cc54-da58-4e0a-ac96-4a9e700ab0c7"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:55.43628502Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:351","msg":"ClusterQueue create event","replica-role":"leader","clusterQueue":{"name":"cq-mcrfz"}}
{"level":"Level(-2)","ts":"2026-11-02T09:45:55.436485694Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-mcrfz","reconcileID":"1c6ab954-e37b-42ce-a65c-ff050bfdd7f0"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:55.457217426Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:240","msg":"LocalQueue create event","replica-role":"leader","localQueue":{"name":"lq-q4ktf","namespace":"test-ns-prfsh"}}
{"level":"Level(-2)","ts":"2026-11-02T09:45:55.457347134Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-prfsh","name":"lq-q4ktf","reconcileID":"fffc17d0-9ee5-494f-8fae-b826835a05bc"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:55.49371849Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-mcrfz"}}
{"level":"Level(-2)","ts":"2026-11-02T09:45:55.494001492Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-mcrfz","reconcileID":"0570db08-0a69-4ac2-a507-9a2f63501c00"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:55.513319422Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-q4ktf","namespace":"test-ns-prfsh"}}
{"level":"Level(-2)","ts":"2026-11-02T09:45:55.513395374Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-prfsh","name":"lq-q4ktf","reconcileID":"2a473343-f3fd-4034-b644-d535716347c9"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:55.571891969Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-q4ktf","namespace":"test-ns-prfsh"}}
{"level":"Level(-2)","ts":"2026-11-02T09:45:55.571974304Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-prfsh","name":"lq-q4ktf","reconcileID":"8616271b-86cb-4c85-a629-4157bc25df60"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:55.572122415Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-prfsh","name":"lq-q4ktf","reconcileID":"482a16c5-e833-4233-a954-f9b3f6bda6c5"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:55.632815853Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-prfsh","name":"test-preemption-trainjob-mgnjr","reconcileID":"e5e249e2-ab2a-4886-964f-1b1b1b404c72","job":"test-ns-prfsh/test-preemption-trainjob-mgnjr","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:55.869436485Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-wdbkl","name":"pod-dw-deployment-7fjs6-86bddc58fc-h2x2t-85c32","reconcileID":"f4a1bba0-6792-49ef-9275-44e58be601cd"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:55.951526511Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1276","msg":"Workload create event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-mgnjr-7b1cc","namespace":"test-ns-prfsh"},"queue":"lq-q4ktf","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:55.951530824Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-prfsh","name":"trainjob-test-preemption-trainjob-mgnjr-7b1cc","reconcileID":"531e7c2f-37ff-42d5-a23e-034e331fa2a6"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:55.951711433Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-prfsh","name":"trainjob-test-preemption-trainjob-mgnjr-7b1cc","reconcileID":"531e7c2f-37ff-42d5-a23e-034e331fa2a6","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T09:45:55.951776704Z","logger":"scheduler","caller":"scheduler/scheduler.go:308","msg":"Obtained heads","replica-role":"leader","schedulingCycle":3,"headCount":1,"waitDuration":42.279713973}
{"level":"Level(-2)","ts":"2026-11-02T09:45:55.951774408Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-prfsh","name":"trainjob-test-preemption-trainjob-mgnjr-7b1cc","reconcileID":"93d52d7a-dec3-49d6-b535-906b3e66ea87"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:55.951824622Z","logger":"scheduler","caller":"scheduler/scheduler.go:323","msg":"Snapshot taken","replica-role":"leader","schedulingCycle":3,"duration":0.000013276}
{"level":"debug","ts":"2026-11-02T09:45:55.951823016Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"test-preemption-trainjob-mgnjr","namespace":"test-ns-prfsh"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"CreatedWorkload","action":"CreatedWorkload","note":"Created Workload: test-ns-prfsh/trainjob-test-preemption-trainjob-mgnjr-7b1cc"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:55.951797268Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-prfsh","name":"test-preemption-trainjob-mgnjr","reconcileID":"691b13c8-0d3c-4c3a-ba3e-8c5c8e7433c1","job":"test-ns-prfsh/test-preemption-trainjob-mgnjr","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:55.95191061Z","logger":"scheduler","caller":"scheduler/scheduler.go:328","msg":"Nomination done","replica-role":"leader","schedulingCycle":3,"entries":1,"inadmissibleEntries":0,"duration":0.000073476}
{"level":"Level(-2)","ts":"2026-11-02T09:45:55.951939858Z","logger":"scheduler","caller":"scheduler/scheduler.go:406","msg":"Attempting to schedule workload","replica-role":"leader","schedulingCycle":3,"workload":{"name":"trainjob-test-preemption-trainjob-mgnjr-7b1cc","namespace":"test-ns-prfsh"},"clusterQueue":{"name":"cq-mcrfz"}}
{"level":"Level(-2)","ts":"2026-11-02T09:45:55.952030924Z","logger":"scheduler","caller":"scheduler/scheduler.go:1041","msg":"Workload assumed in the cache","replica-role":"leader","schedulingCycle":3,"workload":{"name":"trainjob-test-preemption-trainjob-mgnjr-7b1cc","namespace":"test-ns-prfsh"},"clusterQueue":{"name":"cq-mcrfz"}}
{"level":"Level(-2)","ts":"2026-11-02T09:45:55.952071899Z","logger":"scheduler","caller":"scheduler/scheduler.go:362","msg":"Workload processing done","replica-role":"leader","schedulingCycle":3,"duration":0.000137822}
{"level":"Level(-2)","ts":"2026-11-02T09:45:55.95208842Z","logger":"scheduler","caller":"scheduler/scheduler.go:297","msg":"Scheduling cycle complete","replica-role":"leader","schedulingCycle":3,"duration":42.280026467}
{"level":"Level(-2)","ts":"2026-11-02T09:45:55.952102118Z","logger":"scheduler","caller":"scheduler/scheduler.go:295","msg":"Scheduling cycle starts","replica-role":"leader","schedulingCycle":4}
{"level":"Level(-2)","ts":"2026-11-02T09:45:56.063148716Z","logger":"scheduler","caller":"scheduler/scheduler.go:1000","msg":"Workload successfully admitted and assigned flavors","replica-role":"leader","schedulingCycle":3,"workload":{"name":"trainjob-test-preemption-trainjob-mgnjr-7b1cc","namespace":"test-ns-prfsh"},"clusterQueue":{"name":"cq-mcrfz"},"assignments":[{"name":"node","count":1}]}
{"level":"debug","ts":"2026-11-02T09:45:56.063188047Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-test-preemption-trainjob-mgnjr-7b1cc","namespace":"test-ns-prfsh"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"QuotaReserved","action":"QuotaReserved","note":"Quota reserved in ClusterQueue cq-mcrfz, wait time since queued was 1s"}
{"level":"debug","ts":"2026-11-02T09:45:56.063219932Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-test-preemption-trainjob-mgnjr-7b1cc","namespace":"test-ns-prfsh"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"Admitted","action":"Admitted","note":"Admitted by ClusterQueue cq-mcrfz, wait time since reservation was 1s"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:56.06322906Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-mgnjr-7b1cc","namespace":"test-ns-prfsh"},"queue":"lq-q4ktf","status":"admitted","prevStatus":"pending","clusterQueue":"cq-mcrfz"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:56.063301835Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-prfsh","name":"trainjob-test-preemption-trainjob-mgnjr-7b1cc","reconcileID":"d083d859-d9bc-4374-932c-57b219c3ca4f"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:56.063351074Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-prfsh","name":"trainjob-test-preemption-trainjob-mgnjr-7b1cc","reconcileID":"d083d859-d9bc-4374-932c-57b219c3ca4f","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T09:45:56.063486416Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-prfsh","name":"trainjob-test-preemption-trainjob-mgnjr-7b1cc","reconcileID":"054edb28-df92-4268-b2a8-d066118a7bab"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:56.109574124Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-prfsh","name":"test-preemption-trainjob-mgnjr","reconcileID":"1829a1d6-df32-4ffc-a821-f3c743bbec13","job":"test-ns-prfsh/test-preemption-trainjob-mgnjr","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:56.267025985Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-prfsh","name":"test-preemption-trainjob-mgnjr","reconcileID":"1829a1d6-df32-4ffc-a821-f3c743bbec13","job":"test-ns-prfsh/test-preemption-trainjob-mgnjr","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-3)","ts":"2026-11-02T09:45:56.375635661Z","logger":"trainjob","caller":"jobframework/reconciler.go:594","msg":"Unsuspending job","replica-role":"leader","namespace":"test-ns-prfsh","name":"test-preemption-trainjob-mgnjr","reconcileID":"1829a1d6-df32-4ffc-a821-f3c743bbec13","job":"test-ns-prfsh/test-preemption-trainjob-mgnjr","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-preemption-trainjob-mgnjr\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-3)","ts":"2026-11-02T09:45:56.375712677Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-prfsh","name":"test-preemption-trainjob-mgnjr","reconcileID":"1829a1d6-df32-4ffc-a821-f3c743bbec13","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-preemption-trainjob-mgnjr\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:56.375812951Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-prfsh","name":"test-preemption-trainjob-mgnjr","reconcileID":"0510bc9e-15d5-4214-ac01-3b564fe8187f","job":"test-ns-prfsh/test-preemption-trainjob-mgnjr","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:56.541902708Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-prfsh","name":"test-preemption-trainjob-mgnjr","reconcileID":"0510bc9e-15d5-4214-ac01-3b564fe8187f","job":"test-ns-prfsh/test-preemption-trainjob-mgnjr","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-3)","ts":"2026-11-02T09:45:56.652488546Z","logger":"trainjob","caller":"jobframework/reconciler.go:594","msg":"Unsuspending job","replica-role":"leader","namespace":"test-ns-prfsh","name":"test-preemption-trainjob-mgnjr","reconcileID":"0510bc9e-15d5-4214-ac01-3b564fe8187f","job":"test-ns-prfsh/test-preemption-trainjob-mgnjr","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-preemption-trainjob-mgnjr\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-3)","ts":"2026-11-02T09:45:56.652587591Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-prfsh","name":"test-preemption-trainjob-mgnjr","reconcileID":"0510bc9e-15d5-4214-ac01-3b564fe8187f","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-preemption-trainjob-mgnjr\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:56.652711922Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-prfsh","name":"test-preemption-trainjob-mgnjr","reconcileID":"837e202c-6614-4d50-8a46-171f93197be5","job":"test-ns-prfsh/test-preemption-trainjob-mgnjr","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:56.811275025Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-prfsh","name":"test-preemption-trainjob-mgnjr","reconcileID":"837e202c-6614-4d50-8a46-171f93197be5","job":"test-ns-prfsh/test-preemption-trainjob-mgnjr","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:56.952217125Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-mcrfz","reconcileID":"d863637f-ca7f-4739-9162-00cd0b119f55"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:56.952216678Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-prfsh","name":"lq-q4ktf","reconcileID":"7291068f-44e2-4d71-a701-6ad758d9dbd7"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:57.011145566Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-q4ktf","namespace":"test-ns-prfsh"}}
{"level":"Level(-2)","ts":"2026-11-02T09:45:57.011289134Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-prfsh","name":"lq-q4ktf","reconcileID":"2d7cbebe-6bc8-4f4b-8d5e-5a1e5f39516d"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:57.013906466Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-mcrfz"}}
{"level":"Level(-2)","ts":"2026-11-02T09:45:57.014062743Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-mcrfz","reconcileID":"e9efb6b0-004f-42ab-8c04-9c6b345faa8b"}
{"level":"debug","ts":"2026-11-02T09:45:57.142652289Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"test-preemption-trainjob-mgnjr","namespace":"test-ns-prfsh"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"Started","action":"Started","note":"Admitted by clusterQueue cq-mcrfz"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:57.142725335Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-prfsh","name":"test-preemption-trainjob-mgnjr","reconcileID":"8fff357e-2539-4a65-80d3-384e3d3e8eb6","job":"test-ns-prfsh/test-preemption-trainjob-mgnjr","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:57.300575691Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-prfsh","name":"test-preemption-trainjob-mgnjr","reconcileID":"b4cbe085-ab7a-4edd-9e33-ec63265ff863","job":"test-ns-prfsh/test-preemption-trainjob-mgnjr","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:57.45909256Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-prfsh","name":"test-preemption-trainjob-mgnjr","reconcileID":"8b73cd66-5d4d-40a9-9b17-5004f6355c65","job":"test-ns-prfsh/test-preemption-trainjob-mgnjr","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:57.617829118Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-prfsh","name":"test-preemption-trainjob-mgnjr","reconcileID":"aaa356b9-7122-4557-817c-104e2cbfcfe8","job":"test-ns-prfsh/test-preemption-trainjob-mgnjr","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:57.776253807Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-prfsh","name":"test-preemption-trainjob-mgnjr","reconcileID":"f2669e60-87f3-4d0c-a539-c4f088765684","job":"test-ns-prfsh/test-preemption-trainjob-mgnjr","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:58.501217565Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-prfsh","name":"trainjob-test-preemption-trainjob-mgnjr-7b1cc","reconcileID":"683c8721-daaf-422b-9ab6-92c236cea3cc"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:59.918234219Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-mgnjr-7b1cc","namespace":"test-ns-prfsh"},"queue":"lq-q4ktf","status":"admitted","clusterQueue":"cq-mcrfz"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:59.918315531Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1381","msg":"Workload will not be queued because the workload is not active","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-mgnjr-7b1cc","namespace":"test-ns-prfsh"},"queue":"lq-q4ktf","status":"admitted","clusterQueue":"cq-mcrfz"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:59.918374656Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-prfsh","name":"test-preemption-trainjob-mgnjr","reconcileID":"264f6128-c412-4cb7-8e29-0aed98b0f8a8","job":"test-ns-prfsh/test-preemption-trainjob-mgnjr","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:59.918277239Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-prfsh","name":"trainjob-test-preemption-trainjob-mgnjr-7b1cc","reconcileID":"47fac81a-8141-46f1-9e2e-04a067f07d18"}
{"level":"Level(-2)","ts":"2026-11-02T09:45:59.918440783Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-prfsh","name":"trainjob-test-preemption-trainjob-mgnjr-7b1cc","reconcileID":"47fac81a-8141-46f1-9e2e-04a067f07d18","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T09:45:59.918419259Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-prfsh","name":"trainjob-test-preemption-trainjob-mgnjr-7b1cc","reconcileID":"1ec086fd-711b-40e5-9f02-9da080e69a51"}
{"level":"debug","ts":"2026-11-02T09:46:00.032796782Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-test-preemption-trainjob-mgnjr-7b1cc","namespace":"test-ns-prfsh"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"EvictedDueToDeactivated","action":"EvictedDueToDeactivated","note":"The workload is deactivated"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:00.032841937Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-prfsh","name":"trainjob-test-preemption-trainjob-mgnjr-7b1cc","reconcileID":"d61501f1-b2a3-4649-8c99-8687180f65b3"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:00.032867088Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-mgnjr-7b1cc","namespace":"test-ns-prfsh"},"queue":"lq-q4ktf","status":"admitted","clusterQueue":"cq-mcrfz"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:00.032906617Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1381","msg":"Workload will not be queued because the workload is not active","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-mgnjr-7b1cc","namespace":"test-ns-prfsh"},"queue":"lq-q4ktf","status":"admitted","clusterQueue":"cq-mcrfz"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:00.032879625Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-prfsh","name":"trainjob-test-preemption-trainjob-mgnjr-7b1cc","reconcileID":"d61501f1-b2a3-4649-8c99-8687180f65b3","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T09:46:00.033006985Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-prfsh","name":"trainjob-test-preemption-trainjob-mgnjr-7b1cc","reconcileID":"ca3ca662-06a6-4786-b8f1-3fddf5d47143"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:00.077806865Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-prfsh","name":"test-preemption-trainjob-mgnjr","reconcileID":"a5fc765b-f1dd-4e2a-a4c8-93a6928324b7","job":"test-ns-prfsh/test-preemption-trainjob-mgnjr","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:00.361330445Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-prfsh","name":"trainjob-test-preemption-trainjob-mgnjr-7b1cc","reconcileID":"59645df0-cc11-4a85-a7b5-218a5f41ed82"}
{"level":"error","ts":"2026-11-02T09:46:00.402574924Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-prfsh","name":"test-preemption-trainjob-mgnjr","reconcileID":"a5fc765b-f1dd-4e2a-a4c8-93a6928324b7","error":"jobs are still active","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"debug","ts":"2026-11-02T09:46:00.402618727Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"test-preemption-trainjob-mgnjr","namespace":"test-ns-prfsh"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"Stopped","action":"Stopped","note":"The workload is deactivated"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:00.402693243Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-prfsh","name":"test-preemption-trainjob-mgnjr","reconcileID":"0c6939ac-3602-439a-b9d7-dd99aa9b193d","job":"test-ns-prfsh/test-preemption-trainjob-mgnjr","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-11-02T09:46:00.560707589Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-prfsh","name":"test-preemption-trainjob-mgnjr","reconcileID":"0c6939ac-3602-439a-b9d7-dd99aa9b193d","error":"jobs are still active","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:00.560835603Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-prfsh","name":"test-preemption-trainjob-mgnjr","reconcileID":"666727b4-7777-4a02-a862-8b1b82529b26","job":"test-ns-prfsh/test-preemption-trainjob-mgnjr","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-11-02T09:46:00.721963573Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-prfsh","name":"test-preemption-trainjob-mgnjr","reconcileID":"666727b4-7777-4a02-a862-8b1b82529b26","error":"jobs are still active","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:00.722155404Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-prfsh","name":"test-preemption-trainjob-mgnjr","reconcileID":"80be88f5-925a-4ed9-9d4e-60da7a5a7f2c","job":"test-ns-prfsh/test-preemption-trainjob-mgnjr","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-11-02T09:46:00.880915767Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-prfsh","name":"test-preemption-trainjob-mgnjr","reconcileID":"80be88f5-925a-4ed9-9d4e-60da7a5a7f2c","error":"jobs are still active","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:00.881082354Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-prfsh","name":"test-preemption-trainjob-mgnjr","reconcileID":"dc0290d1-d3c6-4c8e-b735-16ad1d1824ae","job":"test-ns-prfsh/test-preemption-trainjob-mgnjr","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:00.918644212Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-prfsh","name":"lq-q4ktf","reconcileID":"cdd4537b-e134-4a81-a764-359d515cef73"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:00.91864421Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-mcrfz","reconcileID":"df00d53b-d07e-4cfe-bcec-e56916673779"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:00.977606711Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-q4ktf","namespace":"test-ns-prfsh"}}
{"level":"Level(-2)","ts":"2026-11-02T09:46:00.977830224Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-prfsh","name":"lq-q4ktf","reconcileID":"6341282d-e3ae-45f1-ac2d-d8a823b31d40"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:00.980783617Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-mcrfz"}}
{"level":"Level(-2)","ts":"2026-11-02T09:46:00.980920825Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-mcrfz","reconcileID":"c828becb-9e1d-4b69-a658-b41686449593"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:00.988841318Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-mcrfz"}}
{"level":"Level(-2)","ts":"2026-11-02T09:46:00.988874718Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-prfsh","name":"trainjob-test-preemption-trainjob-mgnjr-7b1cc","reconcileID":"db3ed2f0-5059-455a-b1e1-8cf85631a6b1"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:00.988902976Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-mcrfz","reconcileID":"4ea7351c-0c31-482c-88b4-8aa7d082b328"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:01.008461388Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:171","msg":"ResourceFlavor update event","replica-role":"leader","resourceFlavor":{"name":"rf-fm4cn"}}
{"level":"Level(-2)","ts":"2026-11-02T09:46:01.008557273Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-fm4cn","reconcileID":"d77516fc-acb5-4853-9cc8-1473f25a265e"}
{"level":"error","ts":"2026-11-02T09:46:01.039494004Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-prfsh","name":"test-preemption-trainjob-mgnjr","reconcileID":"dc0290d1-d3c6-4c8e-b735-16ad1d1824ae","error":"jobs are still active","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:01.039627634Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-prfsh","name":"test-preemption-trainjob-mgnjr","reconcileID":"544dea0d-231c-4300-885e-c0ba34a669b1","job":"test-ns-prfsh/test-preemption-trainjob-mgnjr","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:01.115912647Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:376","msg":"ClusterQueue delete event","replica-role":"leader","clusterQueue":{"name":"cq-mcrfz"}}
{"level":"Level(-2)","ts":"2026-11-02T09:46:01.115951316Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-prfsh","name":"lq-q4ktf","reconcileID":"5d86907e-b239-40fb-b70c-74e1ac7755d4"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:01.116000903Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:385","msg":"Cleared resource metrics for deleted ClusterQueue.","replica-role":"leader","clusterQueue":{"name":"cq-mcrfz"}}
{"level":"Level(-2)","ts":"2026-11-02T09:46:01.116013563Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-prfsh","name":"trainjob-test-preemption-trainjob-mgnjr-7b1cc","reconcileID":"e5fe7f60-598b-47e3-a7e7-6c372374c78f"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:01.116132241Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-fm4cn","reconcileID":"7a9fecbd-078f-423c-b25b-dcf5932c5e52"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:01.171493468Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-q4ktf","namespace":"test-ns-prfsh"}}
{"level":"Level(-2)","ts":"2026-11-02T09:46:01.171612347Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-prfsh","name":"lq-q4ktf","reconcileID":"3ea6d5da-eb5a-4213-948f-e6003ed0910d"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:01.226777942Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:159","msg":"ResourceFlavor delete event","replica-role":"leader","resourceFlavor":{"name":"rf-fm4cn"}}
{"level":"Level(-2)","ts":"2026-11-02T09:46:01.335705838Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:141","msg":"ResourceFlavor create event","replica-role":"leader","resourceFlavor":{"name":"rf-pmx8n"}}
{"level":"Level(-2)","ts":"2026-11-02T09:46:01.33580714Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-pmx8n","reconcileID":"5c1acc6f-b0a3-4a2b-aae2-f726835349c6"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:01.461514551Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:351","msg":"ClusterQueue create event","replica-role":"leader","clusterQueue":{"name":"cq-94ctz"}}
{"level":"Level(-2)","ts":"2026-11-02T09:46:01.461724977Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-94ctz","reconcileID":"9503c329-28bc-4ed0-a97c-fb2e1f3353b3"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:01.477175887Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-prfsh","name":"test-preemption-trainjob-mgnjr","reconcileID":"610974ef-2628-4d7a-94b8-ccca5d7dba4c","job":"test-ns-prfsh/test-preemption-trainjob-mgnjr","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:01.477242271Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-mgnjr-7b1cc","namespace":"test-ns-prfsh"},"queue":"lq-q4ktf","status":"pending","prevStatus":"admitted","prevClusterQueue":"cq-mcrfz"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:01.477260659Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-prfsh","name":"trainjob-test-preemption-trainjob-mgnjr-7b1cc","reconcileID":"912d5c42-5f9f-4073-8b94-821ceb618e84"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:01.477299139Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-prfsh","name":"trainjob-test-preemption-trainjob-mgnjr-7b1cc","reconcileID":"912d5c42-5f9f-4073-8b94-821ceb618e84","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T09:46:01.477299125Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1381","msg":"Workload will not be queued because the workload is not active","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-mgnjr-7b1cc","namespace":"test-ns-prfsh"},"queue":"lq-q4ktf","status":"pending","prevStatus":"admitted","prevClusterQueue":"cq-mcrfz"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:01.47737343Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-prfsh","name":"trainjob-test-preemption-trainjob-mgnjr-7b1cc","reconcileID":"aa629870-9573-43e1-8f4a-10f7c2b0a4d6"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:01.479989628Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:240","msg":"LocalQueue create event","replica-role":"leader","localQueue":{"name":"lq-rlhqt","namespace":"test-ns-bv9zz"}}
{"level":"Level(-2)","ts":"2026-11-02T09:46:01.480102667Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-bv9zz","name":"lq-rlhqt","reconcileID":"ad21d058-07c2-49e5-94b7-cbe963371806"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:01.522871296Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-94ctz"}}
{"level":"Level(-2)","ts":"2026-11-02T09:46:01.523038079Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-94ctz","reconcileID":"11dbf432-dfc7-4d34-a1b6-85d506ef91e0"}
{"level":"error","ts":"2026-11-02T09:46:01.529081676Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-prfsh","name":"test-preemption-trainjob-mgnjr","reconcileID":"610974ef-2628-4d7a-94b8-ccca5d7dba4c","job":"test-ns-prfsh/test-preemption-trainjob-mgnjr","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-mgnjr\" is forbidden: unable to create new content in namespace test-ns-prfsh because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-02T09:46:01.529137479Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-prfsh","name":"test-preemption-trainjob-mgnjr","reconcileID":"610974ef-2628-4d7a-94b8-ccca5d7dba4c","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-mgnjr\" is forbidden: unable to create new content in namespace test-ns-prfsh because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:01.529213896Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-prfsh","name":"test-preemption-trainjob-mgnjr","reconcileID":"ff860f4d-b34c-4c60-b731-636228541ffb","job":"test-ns-prfsh/test-preemption-trainjob-mgnjr","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:01.53838863Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-rlhqt","namespace":"test-ns-bv9zz"}}
{"level":"Level(-2)","ts":"2026-11-02T09:46:01.538723346Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-bv9zz","name":"lq-rlhqt","reconcileID":"a22b529d-64c2-4dc1-8dcf-c209b9432378"}
{"level":"error","ts":"2026-11-02T09:46:01.580854879Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-prfsh","name":"test-preemption-trainjob-mgnjr","reconcileID":"ff860f4d-b34c-4c60-b731-636228541ffb","job":"test-ns-prfsh/test-preemption-trainjob-mgnjr","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-mgnjr\" is forbidden: unable to create new content in namespace test-ns-prfsh because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-02T09:46:01.580937844Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-prfsh","name":"test-preemption-trainjob-mgnjr","reconcileID":"ff860f4d-b34c-4c60-b731-636228541ffb","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-mgnjr\" is forbidden: unable to create new content in namespace test-ns-prfsh because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:01.591403824Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-prfsh","name":"test-preemption-trainjob-mgnjr","reconcileID":"71d5c619-f06f-4014-980f-077de9bc4ec0","job":"test-ns-prfsh/test-preemption-trainjob-mgnjr","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:01.602239942Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-mgnjr-7b1cc","namespace":"test-ns-prfsh"},"queue":"lq-q4ktf","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:01.602283528Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-prfsh","name":"trainjob-test-preemption-trainjob-mgnjr-7b1cc","reconcileID":"edf1c29b-ad6e-4c41-9a13-64f56097db06"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:01.602295737Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1381","msg":"Workload will not be queued because the workload is not active","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-mgnjr-7b1cc","namespace":"test-ns-prfsh"},"queue":"lq-q4ktf","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:01.602320926Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-prfsh","name":"trainjob-test-preemption-trainjob-mgnjr-7b1cc","reconcileID":"edf1c29b-ad6e-4c41-9a13-64f56097db06","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T09:46:01.602369823Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-prfsh","name":"trainjob-test-preemption-trainjob-mgnjr-7b1cc","reconcileID":"9ca4b89b-fa8b-4853-8c36-28814622300f"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:01.609755607Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-rlhqt","namespace":"test-ns-bv9zz"}}
{"level":"Level(-2)","ts":"2026-11-02T09:46:01.609866949Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-bv9zz","name":"lq-rlhqt","reconcileID":"794d6ac2-c9a5-43a8-8fbd-dd929b728705"}
{"level":"error","ts":"2026-11-02T09:46:01.643179894Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-prfsh","name":"test-preemption-trainjob-mgnjr","reconcileID":"71d5c619-f06f-4014-980f-077de9bc4ec0","job":"test-ns-prfsh/test-preemption-trainjob-mgnjr","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-mgnjr\" is forbidden: unable to create new content in namespace test-ns-prfsh because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-02T09:46:01.643255799Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-prfsh","name":"test-preemption-trainjob-mgnjr","reconcileID":"71d5c619-f06f-4014-980f-077de9bc4ec0","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-mgnjr\" is forbidden: unable to create new content in namespace test-ns-prfsh because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:01.643343863Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-prfsh","name":"test-preemption-trainjob-mgnjr","reconcileID":"8d74e0b3-14f9-49d9-a127-3049f6ad5647","job":"test-ns-prfsh/test-preemption-trainjob-mgnjr","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-11-02T09:46:01.695347394Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-prfsh","name":"test-preemption-trainjob-mgnjr","reconcileID":"8d74e0b3-14f9-49d9-a127-3049f6ad5647","job":"test-ns-prfsh/test-preemption-trainjob-mgnjr","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-mgnjr\" is forbidden: unable to create new content in namespace test-ns-prfsh because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-02T09:46:01.695414994Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-prfsh","name":"test-preemption-trainjob-mgnjr","reconcileID":"8d74e0b3-14f9-49d9-a127-3049f6ad5647","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-mgnjr\" is forbidden: unable to create new content in namespace test-ns-prfsh because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:01.695516949Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-bv9zz","name":"test-runtimepatches-trainjob-tb5m2","reconcileID":"4a80d6f6-ca60-43f7-afa7-0ac5b28f1ab2","job":"test-ns-bv9zz/test-runtimepatches-trainjob-tb5m2","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:02.014676407Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1276","msg":"Workload create event","replica-role":"leader","workload":{"name":"trainjob-test-runtimepatches-trainjob-tb5m2-8f87d","namespace":"test-ns-bv9zz"},"queue":"lq-rlhqt","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:02.014689416Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-bv9zz","name":"trainjob-test-runtimepatches-trainjob-tb5m2-8f87d","reconcileID":"f85e99e5-2d69-4f56-a3e8-e0168fea427c"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:02.014990474Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-bv9zz","name":"trainjob-test-runtimepatches-trainjob-tb5m2-8f87d","reconcileID":"f85e99e5-2d69-4f56-a3e8-e0168fea427c","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T09:46:02.015011536Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-bv9zz","name":"trainjob-test-runtimepatches-trainjob-tb5m2-8f87d","reconcileID":"41d439eb-d1cb-4153-b7ad-c8e1eabca2a8"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:02.015077557Z","logger":"scheduler","caller":"scheduler/scheduler.go:308","msg":"Obtained heads","replica-role":"leader","schedulingCycle":4,"headCount":1,"waitDuration":6.062974574}
{"level":"Level(-2)","ts":"2026-11-02T09:46:02.015115444Z","logger":"scheduler","caller":"scheduler/scheduler.go:323","msg":"Snapshot taken","replica-role":"leader","schedulingCycle":4,"duration":0.000013303}
{"level":"debug","ts":"2026-11-02T09:46:02.015156413Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"test-runtimepatches-trainjob-tb5m2","namespace":"test-ns-bv9zz"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"CreatedWorkload","action":"CreatedWorkload","note":"Created Workload: test-ns-bv9zz/trainjob-test-runtimepatches-trainjob-tb5m2-8f87d"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:02.015210393Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-prfsh","name":"test-preemption-trainjob-mgnjr","reconcileID":"922cf7eb-ee70-4665-9271-3b6011e02f0b","job":"test-ns-prfsh/test-preemption-trainjob-mgnjr","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:02.015210123Z","logger":"scheduler","caller":"scheduler/scheduler.go:328","msg":"Nomination done","replica-role":"leader","schedulingCycle":4,"entries":1,"inadmissibleEntries":0,"duration":0.000081376}
{"level":"Level(-2)","ts":"2026-11-02T09:46:02.015309298Z","logger":"scheduler","caller":"scheduler/scheduler.go:406","msg":"Attempting to schedule workload","replica-role":"leader","schedulingCycle":4,"workload":{"name":"trainjob-test-runtimepatches-trainjob-tb5m2-8f87d","namespace":"test-ns-bv9zz"},"clusterQueue":{"name":"cq-94ctz"}}
{"level":"Level(-2)","ts":"2026-11-02T09:46:02.015422728Z","logger":"scheduler","caller":"scheduler/scheduler.go:1041","msg":"Workload assumed in the cache","replica-role":"leader","schedulingCycle":4,"workload":{"name":"trainjob-test-runtimepatches-trainjob-tb5m2-8f87d","namespace":"test-ns-bv9zz"},"clusterQueue":{"name":"cq-94ctz"}}
{"level":"Level(-2)","ts":"2026-11-02T09:46:02.01546049Z","logger":"scheduler","caller":"scheduler/scheduler.go:362","msg":"Workload processing done","replica-role":"leader","schedulingCycle":4,"duration":0.00015593}
{"level":"Level(-2)","ts":"2026-11-02T09:46:02.015477543Z","logger":"scheduler","caller":"scheduler/scheduler.go:297","msg":"Scheduling cycle complete","replica-role":"leader","schedulingCycle":4,"duration":6.063375119}
{"level":"Level(-2)","ts":"2026-11-02T09:46:02.015497727Z","logger":"scheduler","caller":"scheduler/scheduler.go:295","msg":"Scheduling cycle starts","replica-role":"leader","schedulingCycle":5}
{"level":"error","ts":"2026-11-02T09:46:02.066974036Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-prfsh","name":"test-preemption-trainjob-mgnjr","reconcileID":"922cf7eb-ee70-4665-9271-3b6011e02f0b","job":"test-ns-prfsh/test-preemption-trainjob-mgnjr","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-mgnjr\" is forbidden: unable to create new content in namespace test-ns-prfsh because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-02T09:46:02.067067485Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-prfsh","name":"test-preemption-trainjob-mgnjr","reconcileID":"922cf7eb-ee70-4665-9271-3b6011e02f0b","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-mgnjr\" is forbidden: unable to create new content in namespace test-ns-prfsh because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:02.067196405Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-bv9zz","name":"test-runtimepatches-trainjob-tb5m2","reconcileID":"352abbb7-0bc1-485c-b070-850fb47c6805","job":"test-ns-bv9zz/test-runtimepatches-trainjob-tb5m2","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:02.127537704Z","logger":"scheduler","caller":"scheduler/scheduler.go:1000","msg":"Workload successfully admitted and assigned flavors","replica-role":"leader","schedulingCycle":4,"workload":{"name":"trainjob-test-runtimepatches-trainjob-tb5m2-8f87d","namespace":"test-ns-bv9zz"},"clusterQueue":{"name":"cq-94ctz"},"assignments":[{"name":"node","flavors":{"cpu":"rf-pmx8n","memory":"rf-pmx8n"},"resourceUsage":{"cpu":"100m","memory":"128Mi"},"count":1}]}
{"level":"debug","ts":"2026-11-02T09:46:02.127582074Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-test-runtimepatches-trainjob-tb5m2-8f87d","namespace":"test-ns-bv9zz"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"QuotaReserved","action":"QuotaReserved","note":"Quota reserved in ClusterQueue cq-94ctz, wait time since queued was 1s"}
{"level":"debug","ts":"2026-11-02T09:46:02.127619815Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-test-runtimepatches-trainjob-tb5m2-8f87d","namespace":"test-ns-bv9zz"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"Admitted","action":"Admitted","note":"Admitted by ClusterQueue cq-94ctz, wait time since reservation was 0s"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:02.127842497Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-bv9zz","name":"trainjob-test-runtimepatches-trainjob-tb5m2-8f87d","reconcileID":"00cd052f-c713-4672-aa8b-158580a28a75"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:02.127888874Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-bv9zz","name":"trainjob-test-runtimepatches-trainjob-tb5m2-8f87d","reconcileID":"00cd052f-c713-4672-aa8b-158580a28a75","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T09:46:02.127883824Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-runtimepatches-trainjob-tb5m2-8f87d","namespace":"test-ns-bv9zz"},"queue":"lq-rlhqt","status":"admitted","prevStatus":"pending","clusterQueue":"cq-94ctz"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:02.128113041Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-bv9zz","name":"trainjob-test-runtimepatches-trainjob-tb5m2-8f87d","reconcileID":"481d9d77-916e-43a2-80ba-1a24855f2a60"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:02.225899765Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-bv9zz","name":"test-runtimepatches-trainjob-tb5m2","reconcileID":"3bb9d4d6-c06e-40b9-9647-54fd84f7cea0","job":"test-ns-bv9zz/test-runtimepatches-trainjob-tb5m2","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:02.385159203Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-bv9zz","name":"test-runtimepatches-trainjob-tb5m2","reconcileID":"3bb9d4d6-c06e-40b9-9647-54fd84f7cea0","job":"test-ns-bv9zz/test-runtimepatches-trainjob-tb5m2","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:02.478518993Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-prfsh","name":"lq-q4ktf","reconcileID":"544517d7-a9cd-4535-84d8-15e7fbb6e519"}
{"level":"Level(-3)","ts":"2026-11-02T09:46:02.494328556Z","logger":"trainjob","caller":"jobframework/reconciler.go:594","msg":"Unsuspending job","replica-role":"leader","namespace":"test-ns-bv9zz","name":"test-runtimepatches-trainjob-tb5m2","reconcileID":"3bb9d4d6-c06e-40b9-9647-54fd84f7cea0","job":"test-ns-bv9zz/test-runtimepatches-trainjob-tb5m2","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-runtimepatches-trainjob-tb5m2\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-3)","ts":"2026-11-02T09:46:02.494407178Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-bv9zz","name":"test-runtimepatches-trainjob-tb5m2","reconcileID":"3bb9d4d6-c06e-40b9-9647-54fd84f7cea0","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-runtimepatches-trainjob-tb5m2\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:02.494497875Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-prfsh","name":"test-preemption-trainjob-mgnjr","reconcileID":"031f0aa6-9a4b-47dd-969b-0c2d22fd09c2","job":"test-ns-prfsh/test-preemption-trainjob-mgnjr","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-11-02T09:46:02.546174876Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-prfsh","name":"test-preemption-trainjob-mgnjr","reconcileID":"031f0aa6-9a4b-47dd-969b-0c2d22fd09c2","job":"test-ns-prfsh/test-preemption-trainjob-mgnjr","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-mgnjr\" is forbidden: unable to create new content in namespace test-ns-prfsh because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-02T09:46:02.546243172Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-prfsh","name":"test-preemption-trainjob-mgnjr","reconcileID":"031f0aa6-9a4b-47dd-969b-0c2d22fd09c2","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-mgnjr\" is forbidden: unable to create new content in namespace test-ns-prfsh because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:02.546373292Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-bv9zz","name":"test-runtimepatches-trainjob-tb5m2","reconcileID":"e3e930e6-0e53-4368-9717-7dab3e652cd4","job":"test-ns-bv9zz/test-runtimepatches-trainjob-tb5m2","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:02.70993941Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-bv9zz","name":"test-runtimepatches-trainjob-tb5m2","reconcileID":"e3e930e6-0e53-4368-9717-7dab3e652cd4","job":"test-ns-bv9zz/test-runtimepatches-trainjob-tb5m2","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:02.750386561Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-94ctz"}}
{"level":"Level(-2)","ts":"2026-11-02T09:46:02.750445418Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-bv9zz","name":"trainjob-test-runtimepatches-trainjob-tb5m2-8f87d","reconcileID":"0c346420-4e48-4ff3-991a-518e06d59c0b"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:02.750475953Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-94ctz","reconcileID":"0b1ded47-f58e-4c33-8bbb-f02c8d96e4f9"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:02.768757917Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:171","msg":"ResourceFlavor update event","replica-role":"leader","resourceFlavor":{"name":"rf-pmx8n"}}
{"level":"Level(-2)","ts":"2026-11-02T09:46:02.768855781Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-pmx8n","reconcileID":"e8661538-fc09-4927-b470-f7bc6e6d2ef8"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:02.809676102Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-94ctz"}}
{"level":"Level(-2)","ts":"2026-11-02T09:46:02.809757826Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-bv9zz","name":"lq-rlhqt","reconcileID":"c54ff722-e524-48e3-ab71-d1d9e42f60eb"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:02.809768478Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-94ctz","reconcileID":"d3621748-809d-4bb5-ac6e-2923209cb911"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:02.809785522Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-bv9zz","name":"trainjob-test-runtimepatches-trainjob-tb5m2-8f87d","reconcileID":"d8dd84b4-e5b6-4477-94ad-c453e3d28076"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:02.866136261Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-rlhqt","namespace":"test-ns-bv9zz"}}
{"level":"Level(-2)","ts":"2026-11-02T09:46:02.866251542Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-bv9zz","name":"lq-rlhqt","reconcileID":"3a9ac89e-52aa-4a80-9c73-e6604ae53988"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:03.04689204Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-prfsh","name":"test-preemption-trainjob-mgnjr","reconcileID":"f20ef18b-8620-4d0c-a627-6bdcb9cbb3a0","job":"test-ns-prfsh/test-preemption-trainjob-mgnjr","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"debug","ts":"2026-11-02T09:46:03.046892726Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"test-runtimepatches-trainjob-tb5m2","namespace":"test-ns-bv9zz"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"Started","action":"Started","note":"Admitted by clusterQueue cq-94ctz"}
{"level":"error","ts":"2026-11-02T09:46:03.098526493Z","logger":"events","caller":"events/event_broadcaster.go:270","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-bv9zz","name":"test-runtimepatches-trainjob-tb5m2.18e4314e1602d3aa"},"error":"events.events.k8s.io \"test-runtimepatches-trainjob-tb5m2.18e4314e1602d3aa\" is forbidden: unable to create new content in namespace test-ns-bv9zz because it is being terminated","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/workspace/upstream/kueue/src/vendor/k8s.io/client-go/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).attemptRecording\n\t/workspace/upstream/kueue/src/vendor/k8s.io/client-go/tools/events/event_broadcaster.go:211\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).recordToSink.func1\n\t/workspace/upstream/kueue/src/vendor/k8s.io/client-go/tools/events/event_broadcaster.go:200"}
{"level":"error","ts":"2026-11-02T09:46:03.098846116Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-prfsh","name":"test-preemption-trainjob-mgnjr","reconcileID":"f20ef18b-8620-4d0c-a627-6bdcb9cbb3a0","job":"test-ns-prfsh/test-preemption-trainjob-mgnjr","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-mgnjr\" is forbidden: unable to create new content in namespace test-ns-prfsh because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-02T09:46:03.098900524Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-prfsh","name":"test-preemption-trainjob-mgnjr","reconcileID":"f20ef18b-8620-4d0c-a627-6bdcb9cbb3a0","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-mgnjr\" is forbidden: unable to create new content in namespace test-ns-prfsh because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:03.09901061Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-bv9zz","name":"test-runtimepatches-trainjob-tb5m2","reconcileID":"dac16f41-d1f4-42c1-b524-40f8451b22fe","job":"test-ns-bv9zz/test-runtimepatches-trainjob-tb5m2","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-11-02T09:46:03.151425912Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-bv9zz","name":"test-runtimepatches-trainjob-tb5m2","reconcileID":"dac16f41-d1f4-42c1-b524-40f8451b22fe","job":"test-ns-bv9zz/test-runtimepatches-trainjob-tb5m2","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-runtimepatches-trainjob-tb5m2\" is forbidden: unable to create new content in namespace test-ns-bv9zz because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-02T09:46:03.151496184Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-bv9zz","name":"test-runtimepatches-trainjob-tb5m2","reconcileID":"dac16f41-d1f4-42c1-b524-40f8451b22fe","error":"workloads.kueue.x-k8s.io \"test-runtimepatches-trainjob-tb5m2\" is forbidden: unable to create new content in namespace test-ns-bv9zz because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:03.157620066Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-bv9zz","name":"test-runtimepatches-trainjob-tb5m2","reconcileID":"5d08684b-1ae5-4ce5-9923-c6f425049954","job":"test-ns-bv9zz/test-runtimepatches-trainjob-tb5m2","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-11-02T09:46:03.210152862Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-bv9zz","name":"test-runtimepatches-trainjob-tb5m2","reconcileID":"5d08684b-1ae5-4ce5-9923-c6f425049954","job":"test-ns-bv9zz/test-runtimepatches-trainjob-tb5m2","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-runtimepatches-trainjob-tb5m2\" is forbidden: unable to create new content in namespace test-ns-bv9zz because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-02T09:46:03.210221156Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-bv9zz","name":"test-runtimepatches-trainjob-tb5m2","reconcileID":"5d08684b-1ae5-4ce5-9923-c6f425049954","error":"workloads.kueue.x-k8s.io \"test-runtimepatches-trainjob-tb5m2\" is forbidden: unable to create new content in namespace test-ns-bv9zz because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T09:46:03.220504842Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-bv9zz","name":"test-runtimepatches-trainjob-tb5m2","reconcileID":"6ca3baff-47f1-492a-9cca-b4fb8e39d496","job":"test-ns-bv9zz/test-runtimepatches-trainjob-tb5m2","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}