{"level":"Level(-2)","ts":"2026-10-31T12:58:56.084473026Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-invalid","reconcileID":"e5060c4a-f312-4b60-bf63-b1b248a690b5","job":"rhai-e2e-progression-77vvt/progression-invalid","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T12:58:56.238244119Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-invalid","reconcileID":"178dedf4-279a-4d70-9ce6-bf2d994639eb","job":"rhai-e2e-progression-77vvt/progression-invalid","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T12:58:56.692395727Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-invalid","reconcileID":"831a438a-e03d-4e70-ae70-be158aae1228","job":"rhai-e2e-progression-77vvt/progression-invalid","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:25.346312469Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-job-failure","reconcileID":"f3b1f635-92f7-459d-9f31-d8c12502a384","job":"rhai-e2e-progression-77vvt/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:25.743451963Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-job-failure","reconcileID":"8502d10c-81e3-4e32-aac0-80e3acb963dc","job":"rhai-e2e-progression-77vvt/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:26.038004647Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-job-failure","reconcileID":"de5f4d6b-4e1b-46e6-8b90-70ba30ad4876","job":"rhai-e2e-progression-77vvt/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:26.131081894Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-custom-config","reconcileID":"54bd3f70-3eb7-41a9-9fe1-664936b19394","job":"rhai-e2e-progression-77vvt/progression-custom-config","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:26.311431513Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-job-failure","reconcileID":"e8594981-7a8f-4adb-b8be-799f5c2a063a","job":"rhai-e2e-progression-77vvt/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:26.36906464Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-custom-config","reconcileID":"827231fb-a3da-4f72-ae5c-7bd13ee05702","job":"rhai-e2e-progression-77vvt/progression-custom-config","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:28.007280461Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-job-failure","reconcileID":"97f05096-4385-4f7f-8809-443bf9938f87","job":"rhai-e2e-progression-77vvt/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:28.226901359Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-job-failure","reconcileID":"e74bea54-30ef-4016-8f50-4a821274a901","job":"rhai-e2e-progression-77vvt/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:28.446873026Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-job-failure","reconcileID":"76e86ebc-ade9-4744-99e0-494614e955f1","job":"rhai-e2e-progression-77vvt/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:33.396136877Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-job-failure","reconcileID":"0acad1b9-a6b1-44f0-b8fd-f270836f5825","job":"rhai-e2e-progression-77vvt/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:33.564162857Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-job-failure","reconcileID":"2cb14250-7b31-4799-8f5d-22ef72577c00","job":"rhai-e2e-progression-77vvt/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:34.408228242Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-job-failure","reconcileID":"c780eb2c-4327-440f-857b-c22510ecd6b2","job":"rhai-e2e-progression-77vvt/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:34.506221884Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-job-failure","reconcileID":"1812d4e3-c42f-4a6c-8e30-f3bb70283390","job":"rhai-e2e-progression-77vvt/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:34.518794263Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-job-failure","reconcileID":"57af3db8-7838-44e4-a91f-e0a49fc85865","job":"rhai-e2e-progression-77vvt/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:34.995245574Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-job-failure","reconcileID":"3035a627-c37e-42b2-a04f-db5a52b2ddc1","job":"rhai-e2e-progression-77vvt/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:35.161598935Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-job-failure","reconcileID":"41032b85-e2ae-4d4a-b2e7-37cbb99c7ab3","job":"rhai-e2e-progression-77vvt/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:36.445681899Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-no-metrics","reconcileID":"c560074a-c21a-48b9-a530-3566d89b1348","job":"rhai-e2e-progression-77vvt/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:36.86607004Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-no-metrics","reconcileID":"f0617ed2-42b5-4249-a0d1-a4bd6f355ca6","job":"rhai-e2e-progression-77vvt/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:37.094571375Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-no-metrics","reconcileID":"b3cd512f-0908-4e86-91e3-515f5f6739d2","job":"rhai-e2e-progression-77vvt/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:37.43303927Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-no-metrics","reconcileID":"f4ebf294-3e83-4edf-b4c6-ed34948d0e83","job":"rhai-e2e-progression-77vvt/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:37.558876772Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-no-metrics","reconcileID":"f9b1dc4f-92cd-4a9c-abd9-1617e3501cbd","job":"rhai-e2e-progression-77vvt/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:38.659572744Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-job-failure","reconcileID":"0da5d9e5-e5d7-488c-b015-bf535b05e8f7","job":"rhai-e2e-progression-77vvt/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:39.518863199Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-no-metrics","reconcileID":"b54329f0-55d8-4643-ba01-fe84991bd793","job":"rhai-e2e-progression-77vvt/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:39.630118316Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-no-metrics","reconcileID":"832850ac-ddab-4019-ba28-2af0faeb9156","job":"rhai-e2e-progression-77vvt/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:44.54097095Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-no-metrics","reconcileID":"6ae8be4d-e2f4-4a2b-a470-ed793c794642","job":"rhai-e2e-progression-77vvt/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:44.651476643Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-no-metrics","reconcileID":"0e94896f-9a82-41a4-8915-de0b2816e5ac","job":"rhai-e2e-progression-77vvt/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:45.550901853Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-no-metrics","reconcileID":"bde3dc48-6eea-4412-b560-b1f549079ae6","job":"rhai-e2e-progression-77vvt/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:45.614361022Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-no-metrics","reconcileID":"d3f88508-b381-4889-b565-6274878dbad0","job":"rhai-e2e-progression-77vvt/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:45.661518302Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-no-metrics","reconcileID":"629261c9-a17c-4098-a292-06184d5f5257","job":"rhai-e2e-progression-77vvt/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:46.149410835Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-no-metrics","reconcileID":"97e0449c-579c-4b8c-930c-a199208d6533","job":"rhai-e2e-progression-77vvt/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:46.314925647Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-no-metrics","reconcileID":"c259beb8-c591-4a2a-bb73-bccebf33568b","job":"rhai-e2e-progression-77vvt/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:48.171325032Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-custom-config","reconcileID":"a6755507-f8ad-47b7-b7b2-dc02e3bded56","job":"rhai-e2e-progression-77vvt/progression-custom-config","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:48.231246615Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-custom-config","reconcileID":"5b191856-6172-4218-aea6-dd34596cf182","job":"rhai-e2e-progression-77vvt/progression-custom-config","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:49.18694839Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-custom-config","reconcileID":"a19f1561-7144-4e02-8d11-1de4e3b92b9a","job":"rhai-e2e-progression-77vvt/progression-custom-config","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:49.248197541Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-custom-config","reconcileID":"5902544b-0010-401e-bca3-b92434664d9b","job":"rhai-e2e-progression-77vvt/progression-custom-config","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:49.282591486Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-custom-config","reconcileID":"8ea97c8a-be72-4fe6-929a-bf6b53ef7e21","job":"rhai-e2e-progression-77vvt/progression-custom-config","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:49.674387353Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-custom-config","reconcileID":"3469950c-dc27-435e-a4b2-0f13fef22965","job":"rhai-e2e-progression-77vvt/progression-custom-config","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:51.94551066Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-custom-config","reconcileID":"892febc4-97a7-4227-b3d5-55073da293a9","job":"rhai-e2e-progression-77vvt/progression-custom-config","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:52.004735543Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-custom-config","reconcileID":"508172e8-4a9e-4816-8add-2aa42520a376","job":"rhai-e2e-progression-77vvt/progression-custom-config","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:52.008697146Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-disabled","reconcileID":"796351cf-f291-442e-9611-59a42571ffbd","job":"rhai-e2e-progression-77vvt/progression-disabled","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:52.071047665Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-enabled","reconcileID":"61b6c282-b96f-4985-aa54-54b188bb9205","job":"rhai-e2e-progression-77vvt/progression-enabled","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:52.133035479Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-invalid","reconcileID":"7386a7c9-62c6-49a7-ae80-58b97794e3ab","job":"rhai-e2e-progression-77vvt/progression-invalid","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:52.197015197Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-job-failure","reconcileID":"554ab9a0-2862-4e32-b00b-8bdb48c12cc1","job":"rhai-e2e-progression-77vvt/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:52.258069278Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-no-metrics","reconcileID":"05ce38e8-4724-4c40-9308-501ecb2d6d81","job":"rhai-e2e-progression-77vvt/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:52.285812513Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-disabled","reconcileID":"0e04e326-d91b-42e7-a3a0-930910059e4c","job":"rhai-e2e-progression-77vvt/progression-disabled","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:52.287800841Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-custom-config","reconcileID":"a5e899ea-5508-4074-ae4b-6b5f74186e40","job":"rhai-e2e-progression-77vvt/progression-custom-config","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:52.292295135Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-disabled","reconcileID":"86644440-b927-4777-bbd6-639486860553","job":"rhai-e2e-progression-77vvt/progression-disabled","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:52.30019263Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-enabled","reconcileID":"8a09d587-c6d6-4089-845b-a63d855397b7","job":"rhai-e2e-progression-77vvt/progression-enabled","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:52.306028016Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-invalid","reconcileID":"b9b5cc0a-0740-40c5-8578-9b715cf95cbe","job":"rhai-e2e-progression-77vvt/progression-invalid","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:52.310746953Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-job-failure","reconcileID":"e7846b81-833f-4e67-bccc-0344380d424b","job":"rhai-e2e-progression-77vvt/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:52.316540269Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-no-metrics","reconcileID":"eb2919ab-b093-4613-b9f5-2a70c20fbcc7","job":"rhai-e2e-progression-77vvt/progression-no-metrics","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:00:52.746140333Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"rhai-e2e-progression-77vvt","name":"progression-job-failure","reconcileID":"63e986c6-767e-414c-8782-d294066b0cc0","job":"rhai-e2e-progression-77vvt/progression-job-failure","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"rhai-e2e-progression-77vvt"}
{"level":"Level(-2)","ts":"2026-10-31T13:01:38.958536999Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-vx4ft","name":"test-trainjob-ktfjk","reconcileID":"f57ab9c4-3369-4212-83f7-eb2a670b7f35","job":"test-ns-vx4ft/test-trainjob-ktfjk","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-vx4ft"}
{"level":"Level(-2)","ts":"2026-10-31T13:01:39.148108796Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-vx4ft","name":"test-trainjob-ktfjk","reconcileID":"821dacf9-99ec-498f-9086-89ae131f3a6f","job":"test-ns-vx4ft/test-trainjob-ktfjk","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-vx4ft"}
{"level":"Level(-2)","ts":"2026-10-31T13:01:39.374280551Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-vx4ft","name":"test-trainjob-ktfjk","reconcileID":"8b6db16b-d70f-410c-a21b-9c9362fba07e","job":"test-ns-vx4ft/test-trainjob-ktfjk","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-vx4ft"}
{"level":"Level(-2)","ts":"2026-10-31T13:01:39.483070724Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-vx4ft","name":"test-trainjob-ktfjk","reconcileID":"4e9039b8-c3e8-486f-aa8b-77110da9a95a","job":"test-ns-vx4ft/test-trainjob-ktfjk","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-vx4ft"}
{"level":"Level(-2)","ts":"2026-10-31T13:01:39.485788167Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-vx4ft","name":"test-trainjob-ktfjk","reconcileID":"72ce46f4-dc41-41ce-964b-8ef13a3d2f48","job":"test-ns-vx4ft/test-trainjob-ktfjk","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-vx4ft"}
{"level":"Level(-2)","ts":"2026-10-31T13:01:39.595317138Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-vx4ft","name":"test-trainjob-ktfjk","reconcileID":"cbe0e64d-8138-476d-817c-70c949dae700","job":"test-ns-vx4ft/test-trainjob-ktfjk","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-vx4ft"}
{"level":"Level(-2)","ts":"2026-10-31T13:01:41.803703033Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-vx4ft","name":"test-trainjob-ktfjk","reconcileID":"9a06e0d2-de6f-4d9e-91cd-3eeb94408236","job":"test-ns-vx4ft/test-trainjob-ktfjk","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-vx4ft"}
{"level":"Level(-2)","ts":"2026-10-31T13:01:41.916861421Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-vx4ft","name":"test-trainjob-ktfjk","reconcileID":"e8288f25-9bb3-4d95-9e57-81cb4075cb9e","job":"test-ns-vx4ft/test-trainjob-ktfjk","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-vx4ft"}
{"level":"Level(-2)","ts":"2026-10-31T13:01:46.837224284Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-vx4ft","name":"test-trainjob-ktfjk","reconcileID":"c6d07731-c0a1-4aab-a690-e162759aa8c3","job":"test-ns-vx4ft/test-trainjob-ktfjk","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-vx4ft"}
{"level":"Level(-2)","ts":"2026-10-31T13:01:46.948996542Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-vx4ft","name":"test-trainjob-ktfjk","reconcileID":"a9db8d6f-fbdb-4825-8682-702faf5c87bd","job":"test-ns-vx4ft/test-trainjob-ktfjk","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-vx4ft"}
{"level":"Level(-2)","ts":"2026-10-31T13:01:47.84753322Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-vx4ft","name":"test-trainjob-ktfjk","reconcileID":"6c4e3b53-5731-4b3b-8624-3aba55cad86f","job":"test-ns-vx4ft/test-trainjob-ktfjk","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-vx4ft"}
{"level":"Level(-2)","ts":"2026-10-31T13:01:47.912255463Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-vx4ft","name":"test-trainjob-ktfjk","reconcileID":"2f8c0ed1-f647-4d0a-93fd-8b44aeefc0ad","job":"test-ns-vx4ft/test-trainjob-ktfjk","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-vx4ft"}
{"level":"Level(-2)","ts":"2026-10-31T13:01:47.957865187Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-vx4ft","name":"test-trainjob-ktfjk","reconcileID":"5e5e8106-1820-4a91-9e33-1c63674c9eb8","job":"test-ns-vx4ft/test-trainjob-ktfjk","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-vx4ft"}
{"level":"Level(-2)","ts":"2026-10-31T13:01:48.068467989Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-vx4ft","name":"test-trainjob-ktfjk","reconcileID":"078270ff-e3df-4df1-a1a7-40faad651df5","job":"test-ns-vx4ft/test-trainjob-ktfjk","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-vx4ft"}
{"level":"Level(-2)","ts":"2026-10-31T13:01:48.24873551Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-cgwh7","name":"test-trainjob-dtsz5","reconcileID":"b6703f79-3a9f-4228-9266-15a861f8af42","job":"test-ns-cgwh7/test-trainjob-dtsz5","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-cgwh7"}
{"level":"Level(-2)","ts":"2026-10-31T13:01:48.424814069Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-cgwh7","name":"test-trainjob-dtsz5","reconcileID":"d1bcfc3d-9368-4460-a505-a909f8a2fd53","job":"test-ns-cgwh7/test-trainjob-dtsz5","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-cgwh7"}
{"level":"Level(-2)","ts":"2026-10-31T13:01:48.60314424Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-cgwh7","name":"test-trainjob-dtsz5","reconcileID":"3c13e1c3-0d64-409b-82d5-f8796866673f","job":"test-ns-cgwh7/test-trainjob-dtsz5","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-cgwh7"}
{"level":"Level(-2)","ts":"2026-10-31T13:01:48.716855924Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-cgwh7","name":"test-trainjob-dtsz5","reconcileID":"8be78814-cc3b-4387-b836-a7156c9df057","job":"test-ns-cgwh7/test-trainjob-dtsz5","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-cgwh7"}
{"level":"Level(-2)","ts":"2026-10-31T13:01:48.736513934Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-cgwh7","name":"test-trainjob-dtsz5","reconcileID":"24d54adb-36ff-44f2-8f1e-2e34886d43ee","job":"test-ns-cgwh7/test-trainjob-dtsz5","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-cgwh7"}
{"level":"Level(-2)","ts":"2026-10-31T13:01:48.829202735Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-cgwh7","name":"test-trainjob-dtsz5","reconcileID":"414febcb-b8f5-40df-8ebd-c2dd9c0c0921","job":"test-ns-cgwh7/test-trainjob-dtsz5","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-cgwh7"}
{"level":"Level(-2)","ts":"2026-10-31T13:12:54.064429912Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-cgwh7","name":"test-trainjob-dtsz5","reconcileID":"9cc7c3a6-de38-42ff-8ab9-615bc3abdb85","job":"test-ns-cgwh7/test-trainjob-dtsz5","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-cgwh7"}
{"level":"Level(-2)","ts":"2026-10-31T13:12:54.179403681Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-cgwh7","name":"test-trainjob-dtsz5","reconcileID":"ee2709c5-cddb-4161-bc18-9bbc8af549ba","job":"test-ns-cgwh7/test-trainjob-dtsz5","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-cgwh7"}
{"level":"Level(-2)","ts":"2026-10-31T13:12:57.00957313Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-cgwh7","name":"test-trainjob-dtsz5","reconcileID":"594f395e-76e3-46e6-8ad5-70f7edca82a3","job":"test-ns-cgwh7/test-trainjob-dtsz5","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-cgwh7"}
{"level":"Level(-2)","ts":"2026-10-31T13:12:57.119384949Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-cgwh7","name":"test-trainjob-dtsz5","reconcileID":"ede10e41-ff0e-4a3e-9956-7d401ecce8ee","job":"test-ns-cgwh7/test-trainjob-dtsz5","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-cgwh7"}
{"level":"Level(-2)","ts":"2026-10-31T13:12:58.021108241Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-cgwh7","name":"test-trainjob-dtsz5","reconcileID":"75ffbfcc-fc8b-4096-9585-937ff1e0f1df","job":"test-ns-cgwh7/test-trainjob-dtsz5","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-cgwh7"}
{"level":"Level(-2)","ts":"2026-10-31T13:12:58.082644251Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-cgwh7","name":"test-trainjob-dtsz5","reconcileID":"1c8378a4-9429-4ba2-bc48-0e8e49cb3b7f","job":"test-ns-cgwh7/test-trainjob-dtsz5","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-cgwh7"}
{"level":"Level(-2)","ts":"2026-10-31T13:12:58.132887697Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-cgwh7","name":"test-trainjob-dtsz5","reconcileID":"1074a04b-28e8-4515-b4ba-f9642409dccd","job":"test-ns-cgwh7/test-trainjob-dtsz5","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-cgwh7"}
{"level":"Level(-2)","ts":"2026-10-31T13:12:58.24508157Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-cgwh7","name":"test-trainjob-dtsz5","reconcileID":"8155db07-3ac5-40f7-b844-22345f6e3fc7","job":"test-ns-cgwh7/test-trainjob-dtsz5","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-cgwh7"}
{"level":"Level(-2)","ts":"2026-10-31T13:12:59.423291893Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-sxx94","name":"test-trainjob-ptv78","reconcileID":"d9da9b71-f547-4b88-b327-fee47f9f4c31","job":"test-ns-sxx94/test-trainjob-ptv78","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-sxx94"}
{"level":"Level(-2)","ts":"2026-10-31T13:12:59.842812647Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-sxx94","name":"test-trainjob-ptv78","reconcileID":"b32a070d-5104-4065-bec9-138e45ed0b8b","job":"test-ns-sxx94/test-trainjob-ptv78","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-sxx94"}
{"level":"Level(-2)","ts":"2026-10-31T13:13:00.066918952Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-sxx94","name":"test-trainjob-ptv78","reconcileID":"6b3be09a-5f44-4551-b860-ee5906039ec1","job":"test-ns-sxx94/test-trainjob-ptv78","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-sxx94"}
{"level":"Level(-2)","ts":"2026-10-31T13:13:00.176758012Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-sxx94","name":"test-trainjob-ptv78","reconcileID":"a3ad9c2d-0b61-412b-8748-765007625a60","job":"test-ns-sxx94/test-trainjob-ptv78","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-sxx94"}
{"level":"Level(-2)","ts":"2026-10-31T13:13:00.404752978Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-sxx94","name":"test-trainjob-ptv78","reconcileID":"ec68c216-c38f-4924-a037-62ed947023e1","job":"test-ns-sxx94/test-trainjob-ptv78","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-sxx94"}
{"level":"Level(-2)","ts":"2026-10-31T13:13:00.514268746Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-sxx94","name":"test-trainjob-ptv78","reconcileID":"63e444f9-d526-4c0a-a06a-db1e8b5339eb","job":"test-ns-sxx94/test-trainjob-ptv78","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-sxx94"}
{"level":"Level(-2)","ts":"2026-10-31T13:15:56.942349533Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-sxx94","name":"test-trainjob-ptv78","reconcileID":"44255194-28d2-40dc-8546-1e6db2a72925","job":"test-ns-sxx94/test-trainjob-ptv78","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-sxx94"}
{"level":"Level(-2)","ts":"2026-10-31T13:15:57.052719092Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-sxx94","name":"test-trainjob-ptv78","reconcileID":"ba93587b-4285-416a-a2af-d151bdedd7c3","job":"test-ns-sxx94/test-trainjob-ptv78","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-sxx94"}
{"level":"Level(-2)","ts":"2026-10-31T13:15:59.947077053Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-sxx94","name":"test-trainjob-ptv78","reconcileID":"39626849-e08c-4cad-90e4-ef979c311784","job":"test-ns-sxx94/test-trainjob-ptv78","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-sxx94"}
{"level":"Level(-2)","ts":"2026-10-31T13:16:00.064789896Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-sxx94","name":"test-trainjob-ptv78","reconcileID":"77f28690-569b-4d47-b247-fd31cd21b838","job":"test-ns-sxx94/test-trainjob-ptv78","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-sxx94"}
{"level":"Level(-2)","ts":"2026-10-31T13:16:00.959954321Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-sxx94","name":"test-trainjob-ptv78","reconcileID":"3431135a-ad83-4558-8b95-9621a68932f4","job":"test-ns-sxx94/test-trainjob-ptv78","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-sxx94"}
{"level":"Level(-2)","ts":"2026-10-31T13:16:01.070137339Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-sxx94","name":"test-trainjob-ptv78","reconcileID":"ea83eaf8-98c3-411f-8e23-d44dce3eb021","job":"test-ns-sxx94/test-trainjob-ptv78","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-sxx94"}
{"level":"Level(-2)","ts":"2026-10-31T13:16:01.076204775Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-sxx94","name":"test-trainjob-ptv78","reconcileID":"5f8869a7-1918-42bf-9884-b86941640a0f","job":"test-ns-sxx94/test-trainjob-ptv78","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-sxx94"}
{"level":"Level(-2)","ts":"2026-10-31T13:16:01.180392009Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-sxx94","name":"test-trainjob-ptv78","reconcileID":"88adce54-b276-4205-8c8f-7287368b032f","job":"test-ns-sxx94/test-trainjob-ptv78","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-sxx94"}
{"level":"Level(-2)","ts":"2026-10-31T13:16:02.419784786Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-5hz2m","name":"test-trainjob-lxsxc","reconcileID":"53107132-dbca-41ec-9962-8499c2bef69c","job":"test-ns-5hz2m/test-trainjob-lxsxc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-5hz2m"}
{"level":"Level(-2)","ts":"2026-10-31T13:16:02.853180791Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-5hz2m","name":"test-trainjob-lxsxc","reconcileID":"2ac4d905-a769-46e9-8677-c1455e69624a","job":"test-ns-5hz2m/test-trainjob-lxsxc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-5hz2m"}
{"level":"Level(-2)","ts":"2026-10-31T13:16:03.09375058Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-5hz2m","name":"test-trainjob-lxsxc","reconcileID":"24ce04ce-5fba-41ec-9c6a-ce93841e5930","job":"test-ns-5hz2m/test-trainjob-lxsxc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-5hz2m"}
{"level":"Level(-2)","ts":"2026-10-31T13:16:03.207267593Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-5hz2m","name":"test-trainjob-lxsxc","reconcileID":"c47f5cf4-b5e9-47cd-ae9e-a5ef38ffc93c","job":"test-ns-5hz2m/test-trainjob-lxsxc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-5hz2m"}
{"level":"Level(-2)","ts":"2026-10-31T13:16:03.42997779Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-5hz2m","name":"test-trainjob-lxsxc","reconcileID":"2aed7b4f-3789-4c68-ba67-76d2c5ad2611","job":"test-ns-5hz2m/test-trainjob-lxsxc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-5hz2m"}
{"level":"Level(-2)","ts":"2026-10-31T13:16:03.553140431Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-5hz2m","name":"test-trainjob-lxsxc","reconcileID":"3b3560d4-ab54-49c2-9047-f501eaaf0752","job":"test-ns-5hz2m/test-trainjob-lxsxc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-5hz2m"}
{"level":"Level(-2)","ts":"2026-10-31T13:16:07.268034079Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-cgwh7","name":"test-trainjob-dtsz5","reconcileID":"6222431f-5cad-4fab-8d67-29e3ce528d9d","job":"test-ns-cgwh7/test-trainjob-dtsz5","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-cgwh7"}
{"level":"Level(-2)","ts":"2026-10-31T13:16:07.385711314Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-cgwh7","name":"test-trainjob-dtsz5","reconcileID":"1a88a815-1ea6-4647-bb16-8dab072e9241","job":"test-ns-cgwh7/test-trainjob-dtsz5","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-cgwh7"}
{"level":"Level(-2)","ts":"2026-10-31T13:16:08.449989722Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-cgwh7","name":"test-trainjob-dtsz5","reconcileID":"8bc9cf53-e4e3-40f3-85df-08ecfb3b10db","job":"test-ns-cgwh7/test-trainjob-dtsz5","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-cgwh7"}
{"level":"Level(-2)","ts":"2026-10-31T13:16:17.987701039Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-5hz2m","name":"test-trainjob-lxsxc","reconcileID":"db2936e9-35b1-4398-b934-4265dc8de536","job":"test-ns-5hz2m/test-trainjob-lxsxc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-5hz2m"}
{"level":"Level(-2)","ts":"2026-10-31T13:16:18.101627307Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-5hz2m","name":"test-trainjob-lxsxc","reconcileID":"84cd4b2f-6217-48a9-a328-079d9cc5a2ba","job":"test-ns-5hz2m/test-trainjob-lxsxc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-5hz2m"}
{"level":"Level(-2)","ts":"2026-10-31T13:16:24.017839327Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-5hz2m","name":"test-trainjob-lxsxc","reconcileID":"a41acf25-c633-43e4-82d8-02462badfaa6","job":"test-ns-5hz2m/test-trainjob-lxsxc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-5hz2m"}
{"level":"Level(-2)","ts":"2026-10-31T13:16:24.132153128Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-5hz2m","name":"test-trainjob-lxsxc","reconcileID":"dab4b8d9-fb7e-42ff-bc65-8cbb36af39e3","job":"test-ns-5hz2m/test-trainjob-lxsxc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-5hz2m"}
{"level":"Level(-2)","ts":"2026-10-31T13:16:25.027481366Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-5hz2m","name":"test-trainjob-lxsxc","reconcileID":"40a4abde-61a5-4bf9-a7aa-d19621d03f4d","job":"test-ns-5hz2m/test-trainjob-lxsxc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-5hz2m"}
{"level":"Level(-2)","ts":"2026-10-31T13:16:25.142462028Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-5hz2m","name":"test-trainjob-lxsxc","reconcileID":"e1642451-f8b4-4426-a020-f1f3637bebba","job":"test-ns-5hz2m/test-trainjob-lxsxc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-5hz2m"}
{"level":"Level(-2)","ts":"2026-10-31T13:16:25.255598361Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-5hz2m","name":"test-trainjob-lxsxc","reconcileID":"427a7f32-5d6e-4a81-b03b-a9b5c8162896","job":"test-ns-5hz2m/test-trainjob-lxsxc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-5hz2m"}
{"level":"Level(-2)","ts":"2026-10-31T13:16:25.382368962Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-5hz2m","name":"test-trainjob-lxsxc","reconcileID":"0a40e410-6f75-457f-bd43-f054a7baa504","job":"test-ns-5hz2m/test-trainjob-lxsxc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-5hz2m"}
{"level":"Level(-2)","ts":"2026-10-31T13:16:25.389813275Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-5hz2m","name":"test-trainjob-lxsxc","reconcileID":"3928a9ee-5718-4fe2-baab-f997bd2468bc","job":"test-ns-5hz2m/test-trainjob-lxsxc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-5hz2m"}
{"level":"Level(-2)","ts":"2026-10-31T13:16:25.498691225Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-5hz2m","name":"test-trainjob-lxsxc","reconcileID":"48948ce0-c288-4635-a5aa-7e5e2fb4bed8","job":"test-ns-5hz2m/test-trainjob-lxsxc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-5hz2m"}
{"level":"Level(-2)","ts":"2026-10-31T13:19:18.890377671Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-5hz2m","name":"test-trainjob-lxsxc","reconcileID":"d2bb75af-0414-43e2-a180-199fab6b7958","job":"test-ns-5hz2m/test-trainjob-lxsxc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-5hz2m"}
{"level":"Level(-2)","ts":"2026-10-31T13:19:19.005548257Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-5hz2m","name":"test-trainjob-lxsxc","reconcileID":"2b1fad9e-d213-42d2-8424-1241f0a67c93","job":"test-ns-5hz2m/test-trainjob-lxsxc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-5hz2m"}
{"level":"Level(-2)","ts":"2026-10-31T13:19:29.89134603Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-5hz2m","name":"test-trainjob-lxsxc","reconcileID":"e603aae1-4962-45df-9563-a107e53615cf","job":"test-ns-5hz2m/test-trainjob-lxsxc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-5hz2m"}
{"level":"Level(-2)","ts":"2026-10-31T13:19:30.005678399Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-5hz2m","name":"test-trainjob-lxsxc","reconcileID":"ced4aca8-e7d8-400b-b464-fb6da2db7576","job":"test-ns-5hz2m/test-trainjob-lxsxc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-5hz2m"}
{"level":"Level(-2)","ts":"2026-10-31T13:19:30.900580415Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-5hz2m","name":"test-trainjob-lxsxc","reconcileID":"f007bb96-e6e1-4df2-8441-d18ee10fff52","job":"test-ns-5hz2m/test-trainjob-lxsxc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-5hz2m"}
{"level":"Level(-2)","ts":"2026-10-31T13:19:31.013072847Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-5hz2m","name":"test-trainjob-lxsxc","reconcileID":"28aa1d05-aeed-4e27-814e-1a66230e316f","job":"test-ns-5hz2m/test-trainjob-lxsxc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-5hz2m"}
{"level":"Level(-2)","ts":"2026-10-31T13:19:31.129429611Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-5hz2m","name":"test-trainjob-lxsxc","reconcileID":"74b15b0f-53b8-4415-8f6a-3ee3e03228d2","job":"test-ns-5hz2m/test-trainjob-lxsxc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-5hz2m"}
{"level":"Level(-2)","ts":"2026-10-31T13:19:31.243795235Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-5hz2m","name":"test-trainjob-lxsxc","reconcileID":"6e35e4c4-69de-420d-a01e-f269e8808454","job":"test-ns-5hz2m/test-trainjob-lxsxc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-5hz2m"}
{"level":"Level(-2)","ts":"2026-10-31T13:19:31.470306051Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-5hz2m","name":"test-trainjob-lxsxc","reconcileID":"fd2ebf50-f7dd-4aa0-9af4-3b5ea8b6321b","job":"test-ns-5hz2m/test-trainjob-lxsxc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-5hz2m"}
{"level":"Level(-2)","ts":"2026-10-31T13:19:31.583895823Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-5hz2m","name":"test-trainjob-lxsxc","reconcileID":"ea10f852-5b2f-442b-8f9f-e78f2b1a6ee7","job":"test-ns-5hz2m/test-trainjob-lxsxc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-5hz2m"}
{"level":"Level(-2)","ts":"2026-10-31T13:19:38.066877843Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-5hz2m","name":"test-trainjob-lxsxc","reconcileID":"4ebacd5c-3052-400d-9c43-126b6e4f6c08","job":"test-ns-5hz2m/test-trainjob-lxsxc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-5hz2m"}
{"level":"Level(-2)","ts":"2026-10-31T13:19:38.180447211Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-5hz2m","name":"test-trainjob-lxsxc","reconcileID":"389fd70b-3727-4e64-a2b4-bd0236b87375","job":"test-ns-5hz2m/test-trainjob-lxsxc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-5hz2m"}
{"level":"Level(-2)","ts":"2026-10-31T13:19:38.225559243Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-5hz2m","name":"test-trainjob-lxsxc","reconcileID":"64445b9b-b858-48f9-9e89-1f2394636ef1","job":"test-ns-5hz2m/test-trainjob-lxsxc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-5hz2m"}
{"level":"Level(-2)","ts":"2026-10-31T13:19:38.29411049Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-5hz2m","name":"test-trainjob-lxsxc","reconcileID":"c6e381a0-9832-4c86-965f-c9854f12c4c2","job":"test-ns-5hz2m/test-trainjob-lxsxc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-5hz2m"}
{"level":"Level(-2)","ts":"2026-10-31T13:19:55.420687287Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-hblv6","name":"j8acc7be60b7","reconcileID":"13ec3dbb-6d1a-4611-88fd-ad7c4065a8ca","job":"test-ns-hblv6/j8acc7be60b7","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-hblv6"}
{"level":"Level(-2)","ts":"2026-10-31T13:19:55.843698714Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-hblv6","name":"j8acc7be60b7","reconcileID":"a5502868-8b5d-48a5-805e-106e4a72a825","job":"test-ns-hblv6/j8acc7be60b7","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-hblv6"}
{"level":"Level(-2)","ts":"2026-10-31T13:19:56.071739573Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-hblv6","name":"j8acc7be60b7","reconcileID":"3576cba2-582c-490c-8e9f-553d859ee689","job":"test-ns-hblv6/j8acc7be60b7","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-hblv6"}
{"level":"Level(-2)","ts":"2026-10-31T13:19:56.203871048Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-hblv6","name":"j8acc7be60b7","reconcileID":"b5df03e9-6223-44ce-8683-1685677f1873","job":"test-ns-hblv6/j8acc7be60b7","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-hblv6"}
{"level":"Level(-2)","ts":"2026-10-31T13:19:56.313680565Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-hblv6","name":"j8acc7be60b7","reconcileID":"b09f3ab0-355b-45a8-88b8-db74c446890d","job":"test-ns-hblv6/j8acc7be60b7","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-hblv6"}
{"level":"Level(-2)","ts":"2026-10-31T13:19:56.426921147Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-hblv6","name":"j8acc7be60b7","reconcileID":"1b537754-8772-4dd3-a431-706be4fbbb06","job":"test-ns-hblv6/j8acc7be60b7","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-hblv6"}
{"level":"Level(-2)","ts":"2026-10-31T13:19:58.36050036Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-hblv6","name":"j8acc7be60b7","reconcileID":"24d61473-b4d4-4364-974f-2bf2f8aa338b","job":"test-ns-hblv6/j8acc7be60b7","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-hblv6"}
{"level":"Level(-2)","ts":"2026-10-31T13:19:58.473356204Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-hblv6","name":"j8acc7be60b7","reconcileID":"1a3fcb59-3033-4230-bc78-bee3d8a5d7f8","job":"test-ns-hblv6/j8acc7be60b7","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-hblv6"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:27.255459059Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-hblv6","name":"j8acc7be60b7","reconcileID":"949e1f4a-f181-4fc5-af9d-69688f94cce7","job":"test-ns-hblv6/j8acc7be60b7","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-hblv6"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:27.369413551Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-hblv6","name":"j8acc7be60b7","reconcileID":"04cccb5f-b169-4868-8bba-949f8cc14770","job":"test-ns-hblv6/j8acc7be60b7","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-hblv6"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:28.266944187Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-hblv6","name":"j8acc7be60b7","reconcileID":"b040ff33-c1cc-44c3-b18d-b01c4195bd2f","job":"test-ns-hblv6/j8acc7be60b7","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-hblv6"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:28.368561778Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-hblv6","name":"j8acc7be60b7","reconcileID":"ab3db7c4-756c-4638-a2cd-38635db94d5c","job":"test-ns-hblv6/j8acc7be60b7","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-hblv6"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:28.377809122Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-hblv6","name":"j8acc7be60b7","reconcileID":"4f0a902b-27bc-4d51-82c9-6d72f465e0aa","job":"test-ns-hblv6/j8acc7be60b7","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-hblv6"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:28.532212796Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-hblv6","name":"j8acc7be60b7","reconcileID":"13a7ecb2-4fe7-4a20-9a37-a7a2c19e5afb","job":"test-ns-hblv6/j8acc7be60b7","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-hblv6"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:33.750679496Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:141","msg":"ResourceFlavor create event","replica-role":"leader","resourceFlavor":{"name":"rf-sb9lj"}}
{"level":"Level(-2)","ts":"2026-10-31T13:20:33.750803312Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-sb9lj","reconcileID":"2063c2de-57ea-4a53-885b-91c334459c2e"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:34.117989947Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:351","msg":"ClusterQueue create event","replica-role":"leader","clusterQueue":{"name":"cq-tf99l"}}
{"level":"Level(-2)","ts":"2026-10-31T13:20:34.118252199Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-tf99l","reconcileID":"521cb5cd-8122-4b8b-8d41-3612289a0628"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:34.136505351Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:240","msg":"LocalQueue create event","replica-role":"leader","localQueue":{"name":"lq-5dnb5","namespace":"test-ns-jqqjz"}}
{"level":"Level(-2)","ts":"2026-10-31T13:20:34.136700027Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-jqqjz","name":"lq-5dnb5","reconcileID":"6519254d-7e7f-446a-939c-4eb748192ab5"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:34.182076278Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-tf99l"}}
{"level":"Level(-2)","ts":"2026-10-31T13:20:34.1822575Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-tf99l","reconcileID":"b4e983f2-29ac-448b-ac07-a314cbfa37b4"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:34.193315406Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-5dnb5","namespace":"test-ns-jqqjz"}}
{"level":"Level(-2)","ts":"2026-10-31T13:20:34.231733163Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-jqqjz","name":"lq-5dnb5","reconcileID":"7f272447-ab77-4af1-8a31-1bab9c769835"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:34.292142054Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-5dnb5","namespace":"test-ns-jqqjz"}}
{"level":"Level(-2)","ts":"2026-10-31T13:20:34.292253377Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-jqqjz","name":"lq-5dnb5","reconcileID":"9c426228-0d43-4bda-8d6e-eb98c2ee7d7b"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:35.010313966Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-jqqjz","name":"dw-deployment-fcxbb-754f947979-8frlm","reconcileID":"b279c44f-a8f3-4b04-aeeb-391ac3fcd743","job":"test-ns-jqqjz/dw-deployment-fcxbb-754f947979-8frlm","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:35.063563417Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1.PriorityClass","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"debug","ts":"2026-10-31T13:20:35.519357296Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"dw-deployment-fcxbb-754f947979-8frlm","namespace":"test-ns-jqqjz"},"kind":"Pod","apiVersion":"v1","type":"Normal","reason":"CreatedWorkload","action":"CreatedWorkload","note":"Created Workload: test-ns-jqqjz/pod-dw-deployment-fcxbb-754f947979-8frlm-001fa"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:35.519770928Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1276","msg":"Workload create event","replica-role":"leader","workload":{"name":"pod-dw-deployment-fcxbb-754f947979-8frlm-001fa","namespace":"test-ns-jqqjz"},"queue":"lq-5dnb5","status":"pending"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:35.519850945Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-jqqjz","name":"dw-deployment-fcxbb-754f947979-8frlm","reconcileID":"fff83e94-1826-45b7-a8ec-26b135f20918","job":"test-ns-jqqjz/dw-deployment-fcxbb-754f947979-8frlm","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:35.519864118Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-jqqjz","name":"pod-dw-deployment-fcxbb-754f947979-8frlm-001fa","reconcileID":"1c0eefe3-4529-42ca-993f-46afba610b2e"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:35.519927645Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-jqqjz","name":"pod-dw-deployment-fcxbb-754f947979-8frlm-001fa","reconcileID":"1c0eefe3-4529-42ca-993f-46afba610b2e","isDeleted":false}
{"level":"Level(-2)","ts":"2026-10-31T13:20:35.520033641Z","logger":"scheduler","caller":"scheduler/scheduler.go:308","msg":"Obtained heads","replica-role":"leader","schedulingCycle":1,"headCount":1,"waitDuration":2095.260338421}
{"level":"Level(-2)","ts":"2026-10-31T13:20:35.520065797Z","logger":"scheduler","caller":"scheduler/scheduler.go:323","msg":"Snapshot taken","replica-role":"leader","schedulingCycle":1,"duration":0.000013229}
{"level":"Level(-2)","ts":"2026-10-31T13:20:35.520091154Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-jqqjz","name":"pod-dw-deployment-fcxbb-754f947979-8frlm-001fa","reconcileID":"e69c491e-5816-4b31-a54d-da6423b8bcec"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:35.520183368Z","logger":"scheduler","caller":"scheduler/scheduler.go:328","msg":"Nomination done","replica-role":"leader","schedulingCycle":1,"entries":1,"inadmissibleEntries":0,"duration":0.000105353}
{"level":"Level(-2)","ts":"2026-10-31T13:20:35.520207697Z","logger":"scheduler","caller":"scheduler/scheduler.go:406","msg":"Attempting to schedule workload","replica-role":"leader","schedulingCycle":1,"workload":{"name":"pod-dw-deployment-fcxbb-754f947979-8frlm-001fa","namespace":"test-ns-jqqjz"},"clusterQueue":{"name":"cq-tf99l"}}
{"level":"Level(-2)","ts":"2026-10-31T13:20:35.520300656Z","logger":"scheduler","caller":"scheduler/scheduler.go:1041","msg":"Workload assumed in the cache","replica-role":"leader","schedulingCycle":1,"workload":{"name":"pod-dw-deployment-fcxbb-754f947979-8frlm-001fa","namespace":"test-ns-jqqjz"},"clusterQueue":{"name":"cq-tf99l"}}
{"level":"Level(-2)","ts":"2026-10-31T13:20:35.52033881Z","logger":"scheduler","caller":"scheduler/scheduler.go:362","msg":"Workload processing done","replica-role":"leader","schedulingCycle":1,"duration":0.000138322}
{"level":"Level(-2)","ts":"2026-10-31T13:20:35.520364156Z","logger":"scheduler","caller":"scheduler/scheduler.go:297","msg":"Scheduling cycle complete","replica-role":"leader","schedulingCycle":1,"duration":2095.260669056}
{"level":"Level(-2)","ts":"2026-10-31T13:20:35.520386797Z","logger":"scheduler","caller":"scheduler/scheduler.go:295","msg":"Scheduling cycle starts","replica-role":"leader","schedulingCycle":2}
{"level":"Level(-2)","ts":"2026-10-31T13:20:35.638467667Z","logger":"scheduler","caller":"scheduler/scheduler.go:1000","msg":"Workload successfully admitted and assigned flavors","replica-role":"leader","schedulingCycle":1,"workload":{"name":"pod-dw-deployment-fcxbb-754f947979-8frlm-001fa","namespace":"test-ns-jqqjz"},"clusterQueue":{"name":"cq-tf99l"},"assignments":[{"name":"main","flavors":{"cpu":"rf-sb9lj","memory":"rf-sb9lj"},"resourceUsage":{"cpu":"1","memory":"3Gi"},"count":1}]}
{"level":"debug","ts":"2026-10-31T13:20:35.638472764Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"pod-dw-deployment-fcxbb-754f947979-8frlm-001fa","namespace":"test-ns-jqqjz"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"QuotaReserved","action":"QuotaReserved","note":"Quota reserved in ClusterQueue cq-tf99l, wait time since queued was 1s"}
{"level":"debug","ts":"2026-10-31T13:20:35.638636146Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"pod-dw-deployment-fcxbb-754f947979-8frlm-001fa","namespace":"test-ns-jqqjz"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"Admitted","action":"Admitted","note":"Admitted by ClusterQueue cq-tf99l, wait time since reservation was 1s"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:35.638820098Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-jqqjz","name":"pod-dw-deployment-fcxbb-754f947979-8frlm-001fa","reconcileID":"278f7477-9be4-4c32-8865-835a64a15529"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:35.638873274Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"pod-dw-deployment-fcxbb-754f947979-8frlm-001fa","namespace":"test-ns-jqqjz"},"queue":"lq-5dnb5","status":"admitted","prevStatus":"pending","clusterQueue":"cq-tf99l"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:35.638893166Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-jqqjz","name":"pod-dw-deployment-fcxbb-754f947979-8frlm-001fa","reconcileID":"278f7477-9be4-4c32-8865-835a64a15529","isDeleted":false}
{"level":"Level(-2)","ts":"2026-10-31T13:20:35.638952725Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-jqqjz","name":"dw-deployment-fcxbb-754f947979-8frlm","reconcileID":"2404a340-8f07-459c-b855-8a1eeb453cdc","job":"test-ns-jqqjz/dw-deployment-fcxbb-754f947979-8frlm","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:35.639111271Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-jqqjz","name":"pod-dw-deployment-fcxbb-754f947979-8frlm-001fa","reconcileID":"50d61dfb-ab72-4239-9ae0-fe09e9ba8836"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:35.639163104Z","logger":"v1_pod","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-jqqjz","name":"dw-deployment-fcxbb-754f947979-8frlm","reconcileID":"2404a340-8f07-459c-b855-8a1eeb453cdc","job":"test-ns-jqqjz/dw-deployment-fcxbb-754f947979-8frlm","gvk":"/v1, Kind=Pod"}
{"level":"debug","ts":"2026-10-31T13:20:35.758189318Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"dw-deployment-fcxbb-754f947979-8frlm","namespace":"test-ns-jqqjz"},"kind":"Pod","apiVersion":"v1","type":"Normal","reason":"Started","action":"Started","note":"Admitted by clusterQueue cq-tf99l"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:35.758583577Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-jqqjz","name":"dw-deployment-fcxbb-754f947979-8frlm","reconcileID":"a93d9cfa-1239-4027-920c-877833f7f8d8","job":"test-ns-jqqjz/dw-deployment-fcxbb-754f947979-8frlm","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:35.768458148Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-jqqjz","name":"dw-deployment-fcxbb-754f947979-8frlm","reconcileID":"d4d44508-e889-4bf2-bff0-82842815a87b","job":"test-ns-jqqjz/dw-deployment-fcxbb-754f947979-8frlm","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:35.831591385Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-jqqjz","name":"dw-deployment-fcxbb-754f947979-8frlm","reconcileID":"d445f296-80a8-40ee-ab9d-12569d4f1aba","job":"test-ns-jqqjz/dw-deployment-fcxbb-754f947979-8frlm","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:35.890258065Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-jqqjz","name":"dw-deployment-fcxbb-754f947979-8frlm","reconcileID":"16b86777-af18-4770-8dbf-66a98141e9ad","job":"test-ns-jqqjz/dw-deployment-fcxbb-754f947979-8frlm","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:36.521005733Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-tf99l","reconcileID":"c73678d8-6cfd-4401-9c8e-3fa3a29c8042"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:36.521004804Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-jqqjz","name":"lq-5dnb5","reconcileID":"714d1413-e947-4072-9fc3-43b2d6b3d7de"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:36.577850508Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-5dnb5","namespace":"test-ns-jqqjz"}}
{"level":"Level(-2)","ts":"2026-10-31T13:20:36.577992219Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-jqqjz","name":"lq-5dnb5","reconcileID":"d190393d-5e44-4b13-94d5-1232fb10b3cc"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:36.580713971Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-tf99l"}}
{"level":"Level(-2)","ts":"2026-10-31T13:20:36.580886754Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-tf99l","reconcileID":"5bb86fb4-0686-4f6a-91f3-81d7cb9de6b7"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:36.758859384Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-jqqjz","name":"pod-dw-deployment-fcxbb-754f947979-8frlm-001fa","reconcileID":"e46e312d-e41a-4f79-8ebd-660217840a32"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:36.759111039Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-jqqjz","name":"dw-deployment-fcxbb-754f947979-8frlm","reconcileID":"3eb8a478-a588-4343-820f-e05ba495f45a","job":"test-ns-jqqjz/dw-deployment-fcxbb-754f947979-8frlm","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:37.75964126Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-jqqjz","name":"pod-dw-deployment-fcxbb-754f947979-8frlm-001fa","reconcileID":"f62531f5-6da6-437c-919b-77137e0201c0"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:38.22240854Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-jqqjz","name":"dw-deployment-fcxbb-754f947979-8frlm","reconcileID":"d9133d66-0f8d-4d82-a57b-dade46e1a1cb","job":"test-ns-jqqjz/dw-deployment-fcxbb-754f947979-8frlm","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:39.222671413Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-jqqjz","name":"pod-dw-deployment-fcxbb-754f947979-8frlm-001fa","reconcileID":"fc647ed0-07f6-458d-98e6-cc1a92e71826"}
{"level":"info","ts":"2026-10-31T13:20:50.51412993Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:117","msg":"Validating create","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"z00c94b1fe60","namespace":"test-ns-jqqjz"},"namespace":"test-ns-jqqjz","name":"z00c94b1fe60","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:redhat-ods-applications:trainer-e2e","requestID":"be415d6e-c515-402c-b057-9bd0d51c13ea","replica-role":"leader"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:50.575051331Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-jqqjz","name":"z00c94b1fe60","reconcileID":"9f111a90-5534-42de-802e-a947c316f7dc","job":"test-ns-jqqjz/z00c94b1fe60","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:50.901646267Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-jqqjz","name":"z00c94b1fe60","reconcileID":"a5c81f59-8adc-4f0c-b1c2-cc0d76955a61","job":"test-ns-jqqjz/z00c94b1fe60","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"debug","ts":"2026-10-31T13:20:50.901636349Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"z00c94b1fe60","namespace":"test-ns-jqqjz"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"CreatedWorkload","action":"CreatedWorkload","note":"Created Workload: test-ns-jqqjz/trainjob-z00c94b1fe60-f6df1"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:50.90173419Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1276","msg":"Workload create event","replica-role":"leader","workload":{"name":"trainjob-z00c94b1fe60-f6df1","namespace":"test-ns-jqqjz"},"queue":"lq-5dnb5","status":"pending"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:50.901795403Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-jqqjz","name":"trainjob-z00c94b1fe60-f6df1","reconcileID":"21c970b9-3e22-4b5a-8157-ef3b1d684e04"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:50.901851988Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-jqqjz","name":"trainjob-z00c94b1fe60-f6df1","reconcileID":"21c970b9-3e22-4b5a-8157-ef3b1d684e04","isDeleted":false}
{"level":"Level(-2)","ts":"2026-10-31T13:20:50.901981382Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-jqqjz","name":"trainjob-z00c94b1fe60-f6df1","reconcileID":"249700fa-e292-4375-8e1d-8d19e70342c3"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:50.901971738Z","logger":"scheduler","caller":"scheduler/scheduler.go:308","msg":"Obtained heads","replica-role":"leader","schedulingCycle":2,"headCount":1,"waitDuration":15.381584052}
{"level":"Level(-2)","ts":"2026-10-31T13:20:50.902061181Z","logger":"scheduler","caller":"scheduler/scheduler.go:323","msg":"Snapshot taken","replica-role":"leader","schedulingCycle":2,"duration":0.000012958}
{"level":"Level(-2)","ts":"2026-10-31T13:20:50.902136035Z","logger":"scheduler","caller":"scheduler/scheduler.go:328","msg":"Nomination done","replica-role":"leader","schedulingCycle":2,"entries":1,"inadmissibleEntries":0,"duration":0.000060409}
{"level":"Level(-2)","ts":"2026-10-31T13:20:50.902156604Z","logger":"scheduler","caller":"scheduler/scheduler.go:406","msg":"Attempting to schedule workload","replica-role":"leader","schedulingCycle":2,"workload":{"name":"trainjob-z00c94b1fe60-f6df1","namespace":"test-ns-jqqjz"},"clusterQueue":{"name":"cq-tf99l"}}
{"level":"Level(-2)","ts":"2026-10-31T13:20:50.902235877Z","logger":"scheduler","caller":"scheduler/scheduler.go:1041","msg":"Workload assumed in the cache","replica-role":"leader","schedulingCycle":2,"workload":{"name":"trainjob-z00c94b1fe60-f6df1","namespace":"test-ns-jqqjz"},"clusterQueue":{"name":"cq-tf99l"}}
{"level":"Level(-2)","ts":"2026-10-31T13:20:50.902257137Z","logger":"scheduler","caller":"scheduler/scheduler.go:362","msg":"Workload processing done","replica-role":"leader","schedulingCycle":2,"duration":0.000104598}
{"level":"Level(-2)","ts":"2026-10-31T13:20:50.90226882Z","logger":"scheduler","caller":"scheduler/scheduler.go:297","msg":"Scheduling cycle complete","replica-role":"leader","schedulingCycle":2,"duration":15.381881966}
{"level":"Level(-2)","ts":"2026-10-31T13:20:50.902281195Z","logger":"scheduler","caller":"scheduler/scheduler.go:295","msg":"Scheduling cycle starts","replica-role":"leader","schedulingCycle":3}
{"level":"Level(-2)","ts":"2026-10-31T13:20:51.016406044Z","logger":"scheduler","caller":"scheduler/scheduler.go:1000","msg":"Workload successfully admitted and assigned flavors","replica-role":"leader","schedulingCycle":2,"workload":{"name":"trainjob-z00c94b1fe60-f6df1","namespace":"test-ns-jqqjz"},"clusterQueue":{"name":"cq-tf99l"},"assignments":[{"name":"node","flavors":{"cpu":"rf-sb9lj","memory":"rf-sb9lj"},"resourceUsage":{"cpu":"4","memory":"16Gi"},"count":2}]}
{"level":"debug","ts":"2026-10-31T13:20:51.016461984Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-z00c94b1fe60-f6df1","namespace":"test-ns-jqqjz"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"QuotaReserved","action":"QuotaReserved","note":"Quota reserved in ClusterQueue cq-tf99l, wait time since queued was 1s"}
{"level":"debug","ts":"2026-10-31T13:20:51.016507575Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-z00c94b1fe60-f6df1","namespace":"test-ns-jqqjz"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"Admitted","action":"Admitted","note":"Admitted by ClusterQueue cq-tf99l, wait time since reservation was 1s"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:51.016646356Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-z00c94b1fe60-f6df1","namespace":"test-ns-jqqjz"},"queue":"lq-5dnb5","status":"admitted","prevStatus":"pending","clusterQueue":"cq-tf99l"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:51.016713894Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-jqqjz","name":"trainjob-z00c94b1fe60-f6df1","reconcileID":"2436e4e5-b273-4d35-b66c-e22f945d4820"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:51.016764885Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-jqqjz","name":"trainjob-z00c94b1fe60-f6df1","reconcileID":"2436e4e5-b273-4d35-b66c-e22f945d4820","isDeleted":false}
{"level":"Level(-2)","ts":"2026-10-31T13:20:51.016922108Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-jqqjz","name":"trainjob-z00c94b1fe60-f6df1","reconcileID":"19b5e10b-2942-4bc9-a8e1-315ae4993665"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:51.064893498Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-jqqjz","name":"z00c94b1fe60","reconcileID":"105f886c-d40d-4b0e-a9b4-be89fab147e4","job":"test-ns-jqqjz/z00c94b1fe60","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:51.226226108Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-jqqjz","name":"z00c94b1fe60","reconcileID":"105f886c-d40d-4b0e-a9b4-be89fab147e4","job":"test-ns-jqqjz/z00c94b1fe60","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"info","ts":"2026-10-31T13:20:51.285210577Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"z00c94b1fe60","namespace":"test-ns-jqqjz"},"namespace":"test-ns-jqqjz","name":"z00c94b1fe60","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"b8a17033-5a6f-4c07-a746-1a9e37624484","replica-role":"leader"}
{"level":"info","ts":"2026-10-31T13:20:51.402632085Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"z00c94b1fe60","namespace":"test-ns-jqqjz"},"namespace":"test-ns-jqqjz","name":"z00c94b1fe60","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"a012aa9e-1465-4b12-95ce-7d5ac1e22206","replica-role":"leader"}
{"level":"debug","ts":"2026-10-31T13:20:51.460533114Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"z00c94b1fe60","namespace":"test-ns-jqqjz"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"Started","action":"Started","note":"Admitted by clusterQueue cq-tf99l"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:51.460554345Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-jqqjz","name":"z00c94b1fe60","reconcileID":"bb95577e-ee6f-4b6b-8152-3e1823611253","job":"test-ns-jqqjz/z00c94b1fe60","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:51.622234148Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-jqqjz","name":"z00c94b1fe60","reconcileID":"bb95577e-ee6f-4b6b-8152-3e1823611253","job":"test-ns-jqqjz/z00c94b1fe60","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-3)","ts":"2026-10-31T13:20:51.678636179Z","logger":"trainjob","caller":"jobframework/reconciler.go:594","msg":"Unsuspending job","replica-role":"leader","namespace":"test-ns-jqqjz","name":"z00c94b1fe60","reconcileID":"bb95577e-ee6f-4b6b-8152-3e1823611253","job":"test-ns-jqqjz/z00c94b1fe60","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"z00c94b1fe60\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-3)","ts":"2026-10-31T13:20:51.678715286Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-jqqjz","name":"z00c94b1fe60","reconcileID":"bb95577e-ee6f-4b6b-8152-3e1823611253","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"z00c94b1fe60\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:51.678826865Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-jqqjz","name":"z00c94b1fe60","reconcileID":"1101f5b9-2147-4979-b4e7-8ef5ed824acf","job":"test-ns-jqqjz/z00c94b1fe60","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:51.841884161Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-jqqjz","name":"z00c94b1fe60","reconcileID":"dffd407a-dd3c-4029-8068-95a5eaa37fe0","job":"test-ns-jqqjz/z00c94b1fe60","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:51.902448971Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-tf99l","reconcileID":"65df6dba-679f-45ac-84d8-7acf9264e33d"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:51.902547541Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-jqqjz","name":"lq-5dnb5","reconcileID":"4c99fb07-d71c-4354-b6ac-e8ef84c33a74"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:51.954095469Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1.PartialObjectMetadata","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:51.960413074Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-5dnb5","namespace":"test-ns-jqqjz"}}
{"level":"Level(-2)","ts":"2026-10-31T13:20:51.960597161Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-jqqjz","name":"lq-5dnb5","reconcileID":"f0f2acd8-4042-484e-a288-73ca40c0acfe"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:51.963778304Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-tf99l"}}
{"level":"Level(-2)","ts":"2026-10-31T13:20:51.963908312Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-tf99l","reconcileID":"a8c76d97-a72a-4c20-aea8-2a8e57246ea2"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:52.002422991Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-jqqjz","name":"z00c94b1fe60","reconcileID":"6cb698e7-8d0b-485e-84e1-2df644fa09fc","job":"test-ns-jqqjz/z00c94b1fe60","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:52.054550833Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1.PartialObjectMetadata","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:52.164024612Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-jqqjz","name":"z00c94b1fe60","reconcileID":"95291684-8c69-4bdb-9f76-7965df7a4008","job":"test-ns-jqqjz/z00c94b1fe60","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:52.469475348Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-jqqjz","name":"z00c94b1fe60","reconcileID":"a8c4b1ae-41a5-4410-a38c-527066e5c455","job":"test-ns-jqqjz/z00c94b1fe60","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:52.63231871Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-jqqjz","name":"z00c94b1fe60","reconcileID":"36a9a2bd-6add-45c2-9b07-5648e23046f6","job":"test-ns-jqqjz/z00c94b1fe60","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:53.21966719Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-jqqjz","name":"trainjob-z00c94b1fe60-f6df1","reconcileID":"b7dcd21b-5f3e-48ea-a552-7b8fea19ad6a"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:54.317584676Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-jqqjz","name":"trainjob-z00c94b1fe60-f6df1","reconcileID":"c9e46a6e-a8ab-4dcb-afa1-705b150154d5"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:55.393530926Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-jqqjz","name":"trainjob-z00c94b1fe60-f6df1","reconcileID":"b5be893d-0468-43ee-8105-98e681cbf9e0"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:55.402217343Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-jqqjz","name":"z00c94b1fe60","reconcileID":"f22ab2ad-1649-42f9-908d-77a785a8bfb8","job":"test-ns-jqqjz/z00c94b1fe60","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T13:20:55.564408349Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-jqqjz","name":"z00c94b1fe60","reconcileID":"b06427fe-edfa-4e66-a34e-2e8164858b95","job":"test-ns-jqqjz/z00c94b1fe60","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:23.578990237Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-jqqjz","name":"trainjob-z00c94b1fe60-f6df1","reconcileID":"824c7975-8b1c-41bf-bd9a-dca1d08dbbaa"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:23.601860468Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-jqqjz","name":"z00c94b1fe60","reconcileID":"7212fda6-4a95-45a0-b621-b13ad62757d4","job":"test-ns-jqqjz/z00c94b1fe60","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:23.765181497Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-jqqjz","name":"z00c94b1fe60","reconcileID":"3d36edf0-0ae2-47ec-adc1-201aeaeaf2e7","job":"test-ns-jqqjz/z00c94b1fe60","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:24.612031583Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-jqqjz","name":"z00c94b1fe60","reconcileID":"7bc752ca-c8d0-4f2d-aa2e-9b85afd0ab4c","job":"test-ns-jqqjz/z00c94b1fe60","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:24.651923344Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-jqqjz","name":"trainjob-z00c94b1fe60-f6df1","reconcileID":"ef17d3ed-db9e-4614-9903-85ab40012a7a"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:24.774773232Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-jqqjz","name":"z00c94b1fe60","reconcileID":"dfd7128a-f8c3-4b8e-b6b1-985f70809215","job":"test-ns-jqqjz/z00c94b1fe60","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:24.941971871Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-jqqjz","name":"z00c94b1fe60","reconcileID":"22e8a97c-ccad-409e-a0de-003578a1d084","job":"test-ns-jqqjz/z00c94b1fe60","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"debug","ts":"2026-10-31T13:21:25.225994319Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"z00c94b1fe60","namespace":"test-ns-jqqjz"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"FinishedWorkload","action":"FinishedWorkload","note":"Workload 'test-ns-jqqjz/trainjob-z00c94b1fe60-f6df1' is declared finished"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:25.226274262Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-z00c94b1fe60-f6df1","namespace":"test-ns-jqqjz"},"queue":"lq-5dnb5","status":"finished","prevStatus":"admitted","clusterQueue":"cq-tf99l"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:25.226319015Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-jqqjz","name":"trainjob-z00c94b1fe60-f6df1","reconcileID":"26c2c095-5cce-4345-a067-3cedb1e16b07"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:25.22646521Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-jqqjz","name":"trainjob-z00c94b1fe60-f6df1","reconcileID":"26c2c095-5cce-4345-a067-3cedb1e16b07","isDeleted":false}
{"level":"Level(-2)","ts":"2026-10-31T13:21:25.226520518Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-jqqjz","name":"trainjob-z00c94b1fe60-f6df1","reconcileID":"73bc1b73-e73f-4cc4-9fc4-28afe2fba262"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:25.22656136Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-jqqjz","name":"z00c94b1fe60","reconcileID":"d05d7b52-093c-44dc-bc9c-41f6e9238d14","job":"test-ns-jqqjz/z00c94b1fe60","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"debug","ts":"2026-10-31T13:21:25.388295258Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"z00c94b1fe60","namespace":"test-ns-jqqjz"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"FinishedWorkload","action":"FinishedWorkload","note":"Workload 'test-ns-jqqjz/trainjob-z00c94b1fe60-f6df1' is declared finished"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:25.50642049Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-z00c94b1fe60-f6df1","namespace":"test-ns-jqqjz"},"queue":"lq-5dnb5","status":"finished","clusterQueue":"cq-tf99l"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:25.50645416Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-jqqjz","name":"trainjob-z00c94b1fe60-f6df1","reconcileID":"af75cfde-66e6-4338-a22e-bd09d71557b7"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:25.506514992Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-jqqjz","name":"trainjob-z00c94b1fe60-f6df1","reconcileID":"af75cfde-66e6-4338-a22e-bd09d71557b7","isDeleted":false}
{"level":"Level(-2)","ts":"2026-10-31T13:21:25.50652483Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-jqqjz","name":"z00c94b1fe60","reconcileID":"af3928dc-a46a-4f21-be4b-0d93ae9fff1a","job":"test-ns-jqqjz/z00c94b1fe60","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:25.506617896Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-jqqjz","name":"trainjob-z00c94b1fe60-f6df1","reconcileID":"a8658b7a-7d9d-4f21-aeb1-07df1c38338c"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:25.66165896Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-jqqjz","name":"trainjob-z00c94b1fe60-f6df1","reconcileID":"1ec0584f-aa06-42dd-92dc-7c2794ec53a6"}
{"level":"debug","ts":"2026-10-31T13:21:25.668572779Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"z00c94b1fe60","namespace":"test-ns-jqqjz"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"FinishedWorkload","action":"FinishedWorkload","note":"Workload 'test-ns-jqqjz/trainjob-z00c94b1fe60-f6df1' is declared finished"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:26.227479371Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-tf99l","reconcileID":"32a95c22-fe23-4ebf-ae14-ad389d46d153"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:26.227467166Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-jqqjz","name":"lq-5dnb5","reconcileID":"ec760868-ad46-419d-aced-59adb101de84"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:26.284312606Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-5dnb5","namespace":"test-ns-jqqjz"}}
{"level":"Level(-2)","ts":"2026-10-31T13:21:26.284459032Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-jqqjz","name":"lq-5dnb5","reconcileID":"b803efe5-35e0-40b8-98a8-6618b7d77b91"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:26.287384859Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-tf99l"}}
{"level":"Level(-2)","ts":"2026-10-31T13:21:26.287626378Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-tf99l","reconcileID":"63ec48a1-e1bc-44a7-834e-d729be626915"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:26.671336081Z","logger":"trainjob","caller":"workload/workload.go:1810","msg":"Workload is orphaned; finishing to release quota","replica-role":"leader","namespace":"test-ns-jqqjz","name":"z00c94b1fe60","reconcileID":"9065d1f5-48da-48af-bfa3-14b9defa84bb","job":"test-ns-jqqjz/z00c94b1fe60","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:26.721460242Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-jqqjz","name":"trainjob-z00c94b1fe60-f6df1","reconcileID":"3cc56a4e-178e-45b0-bc44-a644b84a991b"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:26.721566001Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-jqqjz","name":"trainjob-z00c94b1fe60-f6df1","reconcileID":"3cc56a4e-178e-45b0-bc44-a644b84a991b","isDeleted":true}
{"level":"Level(-2)","ts":"2026-10-31T13:21:26.721510557Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1312","msg":"Workload delete event","replica-role":"leader","workload":{"name":"trainjob-z00c94b1fe60-f6df1","namespace":"test-ns-jqqjz"},"queue":"lq-5dnb5","status":"finished"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:27.722624428Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-jqqjz","name":"lq-5dnb5","reconcileID":"5c412c7e-356b-4675-a21e-4d5ef61b535c"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:27.722624405Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-tf99l","reconcileID":"5b4d131c-fb4a-40c2-bb24-f1bda648c893"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:28.0017601Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-tf99l"}}
{"level":"Level(-2)","ts":"2026-10-31T13:21:28.001850131Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-tf99l","reconcileID":"c996b28c-668a-4860-b010-ddfebb1f9db9"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:28.001864828Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-jqqjz","name":"pod-dw-deployment-fcxbb-754f947979-8frlm-001fa","reconcileID":"80933322-8202-4daf-8aa3-052ef59c5bb6"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:28.02084716Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:171","msg":"ResourceFlavor update event","replica-role":"leader","resourceFlavor":{"name":"rf-sb9lj"}}
{"level":"Level(-2)","ts":"2026-10-31T13:21:28.020962535Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-sb9lj","reconcileID":"9025f8bb-0a39-4c8e-a6de-bbedc85579fe"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:28.061521128Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-tf99l"}}
{"level":"Level(-2)","ts":"2026-10-31T13:21:28.061586429Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-jqqjz","name":"lq-5dnb5","reconcileID":"df4d2844-d82a-4bc3-90b4-55fa298eb47b"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:28.061709611Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-jqqjz","name":"pod-dw-deployment-fcxbb-754f947979-8frlm-001fa","reconcileID":"b3bfe7ce-4a6e-4d48-b564-758133199239"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:28.06159658Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-tf99l","reconcileID":"371a046c-e9ab-4542-8e9a-1d86dede71fa"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:28.120306184Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-5dnb5","namespace":"test-ns-jqqjz"}}
{"level":"Level(-2)","ts":"2026-10-31T13:21:28.120416553Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-jqqjz","name":"lq-5dnb5","reconcileID":"8bc51159-3314-47ff-a781-0a2715297dfa"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:28.368167069Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:141","msg":"ResourceFlavor create event","replica-role":"leader","resourceFlavor":{"name":"rf-rpcgk"}}
{"level":"Level(-2)","ts":"2026-10-31T13:21:28.36825939Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-rpcgk","reconcileID":"90571256-c33f-4c4f-a684-d51386adfcea"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:28.493746319Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:351","msg":"ClusterQueue create event","replica-role":"leader","clusterQueue":{"name":"cq-l9rdm"}}
{"level":"Level(-2)","ts":"2026-10-31T13:21:28.493947705Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-l9rdm","reconcileID":"fc323a97-088d-47e5-9c8a-17c4a80e5295"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:28.511438026Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:240","msg":"LocalQueue create event","replica-role":"leader","localQueue":{"name":"lq-k88rx","namespace":"test-ns-qlncb"}}
{"level":"Level(-2)","ts":"2026-10-31T13:21:28.511587191Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-qlncb","name":"lq-k88rx","reconcileID":"9c136e5d-9544-4c62-9de3-9a6d45e0da05"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:28.553028762Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-l9rdm"}}
{"level":"Level(-2)","ts":"2026-10-31T13:21:28.553175742Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-l9rdm","reconcileID":"6ef12227-888a-433f-8f55-3161b2c68e66"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:28.56779346Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-k88rx","namespace":"test-ns-qlncb"}}
{"level":"Level(-2)","ts":"2026-10-31T13:21:28.568005775Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-qlncb","name":"lq-k88rx","reconcileID":"8022ea62-875c-4778-97da-0baf87b3812c"}
{"level":"info","ts":"2026-10-31T13:21:28.580969842Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:117","msg":"Validating create","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-preemption-trainjob-9kdmc","namespace":"test-ns-qlncb"},"namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:admin","requestID":"6010da54-5798-4ce6-a370-f027da54859d","replica-role":"leader"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:28.626522694Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-k88rx","namespace":"test-ns-qlncb"}}
{"level":"Level(-2)","ts":"2026-10-31T13:21:28.626637967Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-qlncb","name":"lq-k88rx","reconcileID":"37a258c5-439a-471c-9c78-07ce5aeac4ec"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:28.634845211Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"7c978322-1815-4bf7-a12f-ed10288d10fe","job":"test-ns-qlncb/test-preemption-trainjob-9kdmc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:28.955679709Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-qlncb","name":"trainjob-test-preemption-trainjob-9kdmc-9cd64","reconcileID":"7e7d52e6-f2d0-49c2-a5aa-cb60f51da68e"}
{"level":"debug","ts":"2026-10-31T13:21:28.955704861Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"test-preemption-trainjob-9kdmc","namespace":"test-ns-qlncb"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"CreatedWorkload","action":"CreatedWorkload","note":"Created Workload: test-ns-qlncb/trainjob-test-preemption-trainjob-9kdmc-9cd64"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:28.955744124Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-qlncb","name":"trainjob-test-preemption-trainjob-9kdmc-9cd64","reconcileID":"7e7d52e6-f2d0-49c2-a5aa-cb60f51da68e","isDeleted":false}
{"level":"Level(-2)","ts":"2026-10-31T13:21:28.955763933Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1276","msg":"Workload create event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-9kdmc-9cd64","namespace":"test-ns-qlncb"},"queue":"lq-k88rx","status":"pending"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:28.955771915Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"3705e1a8-7089-47cd-88ef-40e43ae892c1","job":"test-ns-qlncb/test-preemption-trainjob-9kdmc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:28.955945977Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-qlncb","name":"trainjob-test-preemption-trainjob-9kdmc-9cd64","reconcileID":"c52b691b-4e5a-4045-836a-a246963ff619"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:28.955957375Z","logger":"scheduler","caller":"scheduler/scheduler.go:308","msg":"Obtained heads","replica-role":"leader","schedulingCycle":3,"headCount":1,"waitDuration":38.05367531}
{"level":"Level(-2)","ts":"2026-10-31T13:21:28.955994272Z","logger":"scheduler","caller":"scheduler/scheduler.go:323","msg":"Snapshot taken","replica-role":"leader","schedulingCycle":3,"duration":0.000013676}
{"level":"Level(-2)","ts":"2026-10-31T13:21:28.956086747Z","logger":"scheduler","caller":"scheduler/scheduler.go:328","msg":"Nomination done","replica-role":"leader","schedulingCycle":3,"entries":1,"inadmissibleEntries":0,"duration":0.000081811}
{"level":"Level(-2)","ts":"2026-10-31T13:21:28.956115798Z","logger":"scheduler","caller":"scheduler/scheduler.go:406","msg":"Attempting to schedule workload","replica-role":"leader","schedulingCycle":3,"workload":{"name":"trainjob-test-preemption-trainjob-9kdmc-9cd64","namespace":"test-ns-qlncb"},"clusterQueue":{"name":"cq-l9rdm"}}
{"level":"Level(-2)","ts":"2026-10-31T13:21:28.956177947Z","logger":"scheduler","caller":"scheduler/scheduler.go:1041","msg":"Workload assumed in the cache","replica-role":"leader","schedulingCycle":3,"workload":{"name":"trainjob-test-preemption-trainjob-9kdmc-9cd64","namespace":"test-ns-qlncb"},"clusterQueue":{"name":"cq-l9rdm"}}
{"level":"Level(-2)","ts":"2026-10-31T13:21:28.956193457Z","logger":"scheduler","caller":"scheduler/scheduler.go:362","msg":"Workload processing done","replica-role":"leader","schedulingCycle":3,"duration":0.000083058}
{"level":"Level(-2)","ts":"2026-10-31T13:21:28.956205944Z","logger":"scheduler","caller":"scheduler/scheduler.go:297","msg":"Scheduling cycle complete","replica-role":"leader","schedulingCycle":3,"duration":38.053924556}
{"level":"Level(-2)","ts":"2026-10-31T13:21:28.956219713Z","logger":"scheduler","caller":"scheduler/scheduler.go:295","msg":"Scheduling cycle starts","replica-role":"leader","schedulingCycle":4}
{"level":"Level(-2)","ts":"2026-10-31T13:21:28.98295855Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-jqqjz","name":"pod-dw-deployment-fcxbb-754f947979-8frlm-001fa","reconcileID":"e0ee1ab5-b5fb-49a1-9906-4b2b51c4c404"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:29.068648977Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-qlncb","name":"trainjob-test-preemption-trainjob-9kdmc-9cd64","reconcileID":"d8c30484-5685-4d39-b786-4f77ad643f90"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:29.068701773Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-9kdmc-9cd64","namespace":"test-ns-qlncb"},"queue":"lq-k88rx","status":"admitted","prevStatus":"pending","clusterQueue":"cq-l9rdm"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:29.068719981Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-qlncb","name":"trainjob-test-preemption-trainjob-9kdmc-9cd64","reconcileID":"d8c30484-5685-4d39-b786-4f77ad643f90","isDeleted":false}
{"level":"debug","ts":"2026-10-31T13:21:29.068777375Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-test-preemption-trainjob-9kdmc-9cd64","namespace":"test-ns-qlncb"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"Admitted","action":"Admitted","note":"Admitted by ClusterQueue cq-l9rdm, wait time since reservation was 1s"}
{"level":"debug","ts":"2026-10-31T13:21:29.068808991Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-test-preemption-trainjob-9kdmc-9cd64","namespace":"test-ns-qlncb"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"QuotaReserved","action":"QuotaReserved","note":"Quota reserved in ClusterQueue cq-l9rdm, wait time since queued was 1s"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:29.068836032Z","logger":"scheduler","caller":"scheduler/scheduler.go:1000","msg":"Workload successfully admitted and assigned flavors","replica-role":"leader","schedulingCycle":3,"workload":{"name":"trainjob-test-preemption-trainjob-9kdmc-9cd64","namespace":"test-ns-qlncb"},"clusterQueue":{"name":"cq-l9rdm"},"assignments":[{"name":"node","count":1}]}
{"level":"Level(-2)","ts":"2026-10-31T13:21:29.06886709Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-qlncb","name":"trainjob-test-preemption-trainjob-9kdmc-9cd64","reconcileID":"6d451cbb-8aea-423f-9492-34a13ed3981d"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:29.114377864Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"eddc7d9d-abdb-4e68-a794-63d3dd96ac8b","job":"test-ns-qlncb/test-preemption-trainjob-9kdmc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:29.273348394Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"eddc7d9d-abdb-4e68-a794-63d3dd96ac8b","job":"test-ns-qlncb/test-preemption-trainjob-9kdmc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-3)","ts":"2026-10-31T13:21:29.328401666Z","logger":"trainjob","caller":"jobframework/reconciler.go:594","msg":"Unsuspending job","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"eddc7d9d-abdb-4e68-a794-63d3dd96ac8b","job":"test-ns-qlncb/test-preemption-trainjob-9kdmc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-preemption-trainjob-9kdmc\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-3)","ts":"2026-10-31T13:21:29.328475477Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"eddc7d9d-abdb-4e68-a794-63d3dd96ac8b","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-preemption-trainjob-9kdmc\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:29.32859367Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"ac422d8d-7ff6-4c2c-a123-d4278d536b8a","job":"test-ns-qlncb/test-preemption-trainjob-9kdmc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:29.488112804Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"ac422d8d-7ff6-4c2c-a123-d4278d536b8a","job":"test-ns-qlncb/test-preemption-trainjob-9kdmc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-3)","ts":"2026-10-31T13:21:29.543092452Z","logger":"trainjob","caller":"jobframework/reconciler.go:594","msg":"Unsuspending job","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"ac422d8d-7ff6-4c2c-a123-d4278d536b8a","job":"test-ns-qlncb/test-preemption-trainjob-9kdmc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-preemption-trainjob-9kdmc\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-3)","ts":"2026-10-31T13:21:29.543170315Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"ac422d8d-7ff6-4c2c-a123-d4278d536b8a","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-preemption-trainjob-9kdmc\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:29.543263129Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"290bec73-f4a9-4566-948f-999e29d8d675","job":"test-ns-qlncb/test-preemption-trainjob-9kdmc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:29.702878001Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"290bec73-f4a9-4566-948f-999e29d8d675","job":"test-ns-qlncb/test-preemption-trainjob-9kdmc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"info","ts":"2026-10-31T13:21:29.759228858Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-preemption-trainjob-9kdmc","namespace":"test-ns-qlncb"},"namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"6cd6b022-1d56-4e1b-8198-650b9c703bb5","replica-role":"leader"}
{"level":"info","ts":"2026-10-31T13:21:29.873759214Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-preemption-trainjob-9kdmc","namespace":"test-ns-qlncb"},"namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"c23b0e8a-a91f-4d35-bfe6-ebd959bc4f59","replica-role":"leader"}
{"level":"debug","ts":"2026-10-31T13:21:29.930515548Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"test-preemption-trainjob-9kdmc","namespace":"test-ns-qlncb"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"Started","action":"Started","note":"Admitted by clusterQueue cq-l9rdm"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:29.930544378Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"630e623b-f285-4496-a8df-b7caa0b8108e","job":"test-ns-qlncb/test-preemption-trainjob-9kdmc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:29.95643161Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-l9rdm","reconcileID":"4f2b60ad-eb34-4018-9c2c-0756443b9f95"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:29.956430588Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-qlncb","name":"lq-k88rx","reconcileID":"bf42ab5a-1a2d-4830-bb5e-46298a2048ae"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:30.01296816Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-k88rx","namespace":"test-ns-qlncb"}}
{"level":"Level(-2)","ts":"2026-10-31T13:21:30.013066168Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-qlncb","name":"lq-k88rx","reconcileID":"b0607f43-cd20-48b7-9cf1-f8eec652bc3a"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:30.016008734Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-l9rdm"}}
{"level":"Level(-2)","ts":"2026-10-31T13:21:30.016168658Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-l9rdm","reconcileID":"a53f5c4d-68d6-40d3-8c4b-2f996197b6b4"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:30.0922935Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"029c5e3c-d3f3-4d8d-95c1-a5463b262c50","job":"test-ns-qlncb/test-preemption-trainjob-9kdmc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:30.252217268Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"b3adf55d-aa07-4bab-a001-60e374fa10f9","job":"test-ns-qlncb/test-preemption-trainjob-9kdmc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:30.411802123Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"c5ee524c-9e77-41cd-b782-dfa47ecffc88","job":"test-ns-qlncb/test-preemption-trainjob-9kdmc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:30.571231054Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"821c9f99-9af8-4b2b-b34e-369dbc0ba425","job":"test-ns-qlncb/test-preemption-trainjob-9kdmc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:30.731005513Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"6234f713-0ded-4071-b4c0-5807dcc5fd46","job":"test-ns-qlncb/test-preemption-trainjob-9kdmc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:31.446429771Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-qlncb","name":"trainjob-test-preemption-trainjob-9kdmc-9cd64","reconcileID":"ea543a00-c12e-4af1-8ed9-f0e7f2d9741f"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:31.808771806Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-9kdmc-9cd64","namespace":"test-ns-qlncb"},"queue":"lq-k88rx","status":"admitted","clusterQueue":"cq-l9rdm"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:31.808795939Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-qlncb","name":"trainjob-test-preemption-trainjob-9kdmc-9cd64","reconcileID":"534cc934-bcf5-4ba6-bb30-68a36c6cb46f"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:31.808838778Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-qlncb","name":"trainjob-test-preemption-trainjob-9kdmc-9cd64","reconcileID":"534cc934-bcf5-4ba6-bb30-68a36c6cb46f","isDeleted":false}
{"level":"Level(-2)","ts":"2026-10-31T13:21:31.808841204Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1381","msg":"Workload will not be queued because the workload is not active","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-9kdmc-9cd64","namespace":"test-ns-qlncb"},"queue":"lq-k88rx","status":"admitted","clusterQueue":"cq-l9rdm"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:31.808964201Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-qlncb","name":"trainjob-test-preemption-trainjob-9kdmc-9cd64","reconcileID":"ac1c8254-dc1a-4e2a-8818-3933014002da"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:31.808979955Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"7bc65515-461e-44f4-910d-08e90c029410","job":"test-ns-qlncb/test-preemption-trainjob-9kdmc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"debug","ts":"2026-10-31T13:21:31.923470972Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-test-preemption-trainjob-9kdmc-9cd64","namespace":"test-ns-qlncb"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"EvictedDueToDeactivated","action":"EvictedDueToDeactivated","note":"The workload is deactivated"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:31.923587246Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-9kdmc-9cd64","namespace":"test-ns-qlncb"},"queue":"lq-k88rx","status":"admitted","clusterQueue":"cq-l9rdm"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:31.923600694Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-qlncb","name":"trainjob-test-preemption-trainjob-9kdmc-9cd64","reconcileID":"e3654142-a010-45b9-9029-57f07326e17a"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:31.923629314Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1381","msg":"Workload will not be queued because the workload is not active","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-9kdmc-9cd64","namespace":"test-ns-qlncb"},"queue":"lq-k88rx","status":"admitted","clusterQueue":"cq-l9rdm"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:31.923658104Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-qlncb","name":"trainjob-test-preemption-trainjob-9kdmc-9cd64","reconcileID":"e3654142-a010-45b9-9029-57f07326e17a","isDeleted":false}
{"level":"Level(-2)","ts":"2026-10-31T13:21:31.923711715Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-qlncb","name":"trainjob-test-preemption-trainjob-9kdmc-9cd64","reconcileID":"256bde5b-8b39-45ad-b97c-ba2ffd08548a"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:31.968815345Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"b9d92609-76c3-4698-a838-67d30b4755cf","job":"test-ns-qlncb/test-preemption-trainjob-9kdmc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"info","ts":"2026-10-31T13:21:32.184926099Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-preemption-trainjob-9kdmc","namespace":"test-ns-qlncb"},"namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"cb8e20af-18d3-4407-b0b1-8ece7634a186","replica-role":"leader"}
{"level":"error","ts":"2026-10-31T13:21:32.241745195Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"b9d92609-76c3-4698-a838-67d30b4755cf","error":"jobs are still active","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"debug","ts":"2026-10-31T13:21:32.241763867Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"test-preemption-trainjob-9kdmc","namespace":"test-ns-qlncb"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"Stopped","action":"Stopped","note":"The workload is deactivated"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:32.241872749Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"e962caa5-6d9f-4e0d-b7c3-cb679988972b","job":"test-ns-qlncb/test-preemption-trainjob-9kdmc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-10-31T13:21:32.401691948Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"e962caa5-6d9f-4e0d-b7c3-cb679988972b","error":"jobs are still active","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:32.401792656Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"70334b54-1e8d-4f1f-b61c-d5940f422038","job":"test-ns-qlncb/test-preemption-trainjob-9kdmc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:32.524712015Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-qlncb","name":"trainjob-test-preemption-trainjob-9kdmc-9cd64","reconcileID":"523cd9ea-7e9a-47c7-a9e8-e6840fe40d7e"}
{"level":"error","ts":"2026-10-31T13:21:32.561352412Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"70334b54-1e8d-4f1f-b61c-d5940f422038","error":"jobs are still active","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:32.561450528Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"0a24c818-a656-4ad5-9d88-bf24a79ceefa","job":"test-ns-qlncb/test-preemption-trainjob-9kdmc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-10-31T13:21:32.720868217Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"0a24c818-a656-4ad5-9d88-bf24a79ceefa","error":"jobs are still active","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:32.720979029Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"8f506fc9-8166-4b50-95aa-d47f919c962d","job":"test-ns-qlncb/test-preemption-trainjob-9kdmc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:32.809314105Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-l9rdm","reconcileID":"71d567c7-76b2-400a-bee4-818d0fe952a7"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:32.80931281Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-qlncb","name":"lq-k88rx","reconcileID":"e575f558-af96-4693-b5f3-368b25481f6c"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:32.868007956Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-k88rx","namespace":"test-ns-qlncb"}}
{"level":"Level(-2)","ts":"2026-10-31T13:21:32.868117069Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-qlncb","name":"lq-k88rx","reconcileID":"d8aa57fe-aa71-4a59-aa00-faa9215de944"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:32.870750626Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-l9rdm"}}
{"level":"Level(-2)","ts":"2026-10-31T13:21:32.870883719Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-l9rdm","reconcileID":"a3cd9313-f395-453e-acbd-05ad036580b6"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:32.877542636Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-l9rdm"}}
{"level":"Level(-2)","ts":"2026-10-31T13:21:32.877620792Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-l9rdm","reconcileID":"f8ebcd51-5705-4d72-b848-eb1c6aaf2513"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:32.877629444Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-qlncb","name":"trainjob-test-preemption-trainjob-9kdmc-9cd64","reconcileID":"41a33c55-d10c-4279-a3f8-c23fd7373af5"}
{"level":"error","ts":"2026-10-31T13:21:32.880257369Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"8f506fc9-8166-4b50-95aa-d47f919c962d","error":"jobs are still active","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:32.880337112Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"b86083ba-c807-49d0-82fb-52b50806e16b","job":"test-ns-qlncb/test-preemption-trainjob-9kdmc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:32.896771999Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:171","msg":"ResourceFlavor update event","replica-role":"leader","resourceFlavor":{"name":"rf-rpcgk"}}
{"level":"Level(-2)","ts":"2026-10-31T13:21:32.896851776Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-rpcgk","reconcileID":"78fdcc1e-fc6a-44de-af31-550bc219c36a"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:32.988737794Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:376","msg":"ClusterQueue delete event","replica-role":"leader","clusterQueue":{"name":"cq-l9rdm"}}
{"level":"Level(-2)","ts":"2026-10-31T13:21:32.988753358Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-qlncb","name":"lq-k88rx","reconcileID":"c510d33a-c02d-4f33-987d-aefef199820f"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:32.988807636Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:385","msg":"Cleared resource metrics for deleted ClusterQueue.","replica-role":"leader","clusterQueue":{"name":"cq-l9rdm"}}
{"level":"Level(-2)","ts":"2026-10-31T13:21:32.988823319Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-qlncb","name":"trainjob-test-preemption-trainjob-9kdmc-9cd64","reconcileID":"66b8453e-7335-4286-b3c3-1869f76d0433"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:32.988922767Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-rpcgk","reconcileID":"3e052c8e-58fc-4956-97df-ce718c69f557"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:33.04642764Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-k88rx","namespace":"test-ns-qlncb"}}
{"level":"Level(-2)","ts":"2026-10-31T13:21:33.046542154Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-qlncb","name":"lq-k88rx","reconcileID":"5ec19397-f12d-49c8-85af-832d463f95c7"}
{"level":"info","ts":"2026-10-31T13:21:33.095770359Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-preemption-trainjob-9kdmc","namespace":"test-ns-qlncb"},"namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"2826f534-b8ff-4c23-98ae-4b551118d83f","replica-role":"leader"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:33.09636885Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:159","msg":"ResourceFlavor delete event","replica-role":"leader","resourceFlavor":{"name":"rf-rpcgk"}}
{"level":"Level(-2)","ts":"2026-10-31T13:21:33.211892233Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:141","msg":"ResourceFlavor create event","replica-role":"leader","resourceFlavor":{"name":"rf-d5rfn"}}
{"level":"Level(-2)","ts":"2026-10-31T13:21:33.211971355Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-d5rfn","reconcileID":"73982de3-28c9-46af-81c1-0bd09c3111e7"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:33.265241452Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"85b0972f-e13b-4e25-82a5-f00665fa9865","job":"test-ns-qlncb/test-preemption-trainjob-9kdmc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:33.265463615Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-qlncb","name":"trainjob-test-preemption-trainjob-9kdmc-9cd64","reconcileID":"916fe90d-44c2-4296-a0ce-a2185e49f970"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:33.265544353Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-qlncb","name":"trainjob-test-preemption-trainjob-9kdmc-9cd64","reconcileID":"916fe90d-44c2-4296-a0ce-a2185e49f970","isDeleted":false}
{"level":"Level(-2)","ts":"2026-10-31T13:21:33.265476846Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-9kdmc-9cd64","namespace":"test-ns-qlncb"},"queue":"lq-k88rx","status":"pending","prevStatus":"admitted","prevClusterQueue":"cq-l9rdm"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:33.265591053Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1381","msg":"Workload will not be queued because the workload is not active","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-9kdmc-9cd64","namespace":"test-ns-qlncb"},"queue":"lq-k88rx","status":"pending","prevStatus":"admitted","prevClusterQueue":"cq-l9rdm"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:33.265668092Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-qlncb","name":"trainjob-test-preemption-trainjob-9kdmc-9cd64","reconcileID":"357dd429-ff43-473a-9df5-6a2e91475e46"}
{"level":"error","ts":"2026-10-31T13:21:33.317550471Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"85b0972f-e13b-4e25-82a5-f00665fa9865","job":"test-ns-qlncb/test-preemption-trainjob-9kdmc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-9kdmc\" is forbidden: unable to create new content in namespace test-ns-qlncb because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-10-31T13:21:33.317608526Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"85b0972f-e13b-4e25-82a5-f00665fa9865","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-9kdmc\" is forbidden: unable to create new content in namespace test-ns-qlncb because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:33.317680719Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"1cb1e1cb-3f5b-4ef2-8405-a4d3328516a2","job":"test-ns-qlncb/test-preemption-trainjob-9kdmc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:33.338740217Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:351","msg":"ClusterQueue create event","replica-role":"leader","clusterQueue":{"name":"cq-gd6tf"}}
{"level":"Level(-2)","ts":"2026-10-31T13:21:33.338906291Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-gd6tf","reconcileID":"0522b671-99ef-4730-bb14-ad4293c7a6f3"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:33.356611168Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:240","msg":"LocalQueue create event","replica-role":"leader","localQueue":{"name":"lq-4ddpz","namespace":"test-ns-p7fzd"}}
{"level":"Level(-2)","ts":"2026-10-31T13:21:33.356733106Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-p7fzd","name":"lq-4ddpz","reconcileID":"b8c58a37-571f-4cb1-9a96-574a8dde60fe"}
{"level":"error","ts":"2026-10-31T13:21:33.369849819Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"1cb1e1cb-3f5b-4ef2-8405-a4d3328516a2","job":"test-ns-qlncb/test-preemption-trainjob-9kdmc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-9kdmc\" is forbidden: unable to create new content in namespace test-ns-qlncb because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-10-31T13:21:33.36990327Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"1cb1e1cb-3f5b-4ef2-8405-a4d3328516a2","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-9kdmc\" is forbidden: unable to create new content in namespace test-ns-qlncb because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:33.380188731Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"34a06184-1be4-4ad3-8ef7-3adfe9445c26","job":"test-ns-qlncb/test-preemption-trainjob-9kdmc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:33.380877279Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-9kdmc-9cd64","namespace":"test-ns-qlncb"},"queue":"lq-k88rx","status":"pending"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:33.380926796Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1381","msg":"Workload will not be queued because the workload is not active","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-9kdmc-9cd64","namespace":"test-ns-qlncb"},"queue":"lq-k88rx","status":"pending"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:33.380945321Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-qlncb","name":"trainjob-test-preemption-trainjob-9kdmc-9cd64","reconcileID":"ac098e3f-a1d8-4308-8c6c-7a393fc83be5"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:33.38098942Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-qlncb","name":"trainjob-test-preemption-trainjob-9kdmc-9cd64","reconcileID":"ac098e3f-a1d8-4308-8c6c-7a393fc83be5","isDeleted":false}
{"level":"Level(-2)","ts":"2026-10-31T13:21:33.380999734Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-qlncb","name":"trainjob-test-preemption-trainjob-9kdmc-9cd64","reconcileID":"40bd3d97-a0ae-4c7d-b149-e607ef490700"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:33.402516617Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-gd6tf"}}
{"level":"Level(-2)","ts":"2026-10-31T13:21:33.402644186Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-gd6tf","reconcileID":"e946d550-af64-4301-8342-74424c8e3815"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:33.414732758Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-4ddpz","namespace":"test-ns-p7fzd"}}
{"level":"Level(-2)","ts":"2026-10-31T13:21:33.414789022Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-p7fzd","name":"lq-4ddpz","reconcileID":"ee5aa886-33c4-4ee0-a58f-28fdd456b4bd"}
{"level":"info","ts":"2026-10-31T13:21:33.427176323Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:117","msg":"Validating create","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-runtimepatches-trainjob-kxk7g","namespace":"test-ns-p7fzd"},"namespace":"test-ns-p7fzd","name":"test-runtimepatches-trainjob-kxk7g","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:admin","requestID":"18ad719b-cb32-4668-a8a5-54b58231d1af","replica-role":"leader"}
{"level":"error","ts":"2026-10-31T13:21:33.432636356Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"34a06184-1be4-4ad3-8ef7-3adfe9445c26","job":"test-ns-qlncb/test-preemption-trainjob-9kdmc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-9kdmc\" is forbidden: unable to create new content in namespace test-ns-qlncb because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-10-31T13:21:33.432690601Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"34a06184-1be4-4ad3-8ef7-3adfe9445c26","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-9kdmc\" is forbidden: unable to create new content in namespace test-ns-qlncb because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:33.43275772Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"86dcbb10-4da5-47d3-bdbc-2326e73f2ac9","job":"test-ns-qlncb/test-preemption-trainjob-9kdmc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:33.471648899Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-p7fzd","name":"lq-4ddpz","reconcileID":"0d384946-cf56-4f9b-bc5d-b65d3f2bcda5"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:33.471718421Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-4ddpz","namespace":"test-ns-p7fzd"}}
{"level":"error","ts":"2026-10-31T13:21:33.484929762Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"86dcbb10-4da5-47d3-bdbc-2326e73f2ac9","job":"test-ns-qlncb/test-preemption-trainjob-9kdmc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-9kdmc\" is forbidden: unable to create new content in namespace test-ns-qlncb because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-10-31T13:21:33.484990608Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"86dcbb10-4da5-47d3-bdbc-2326e73f2ac9","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-9kdmc\" is forbidden: unable to create new content in namespace test-ns-qlncb because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:33.485062975Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-p7fzd","name":"test-runtimepatches-trainjob-kxk7g","reconcileID":"2a05eb73-dc62-4260-8bed-eaa0d14c4ce4","job":"test-ns-p7fzd/test-runtimepatches-trainjob-kxk7g","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-3)","ts":"2026-10-31T13:21:33.525546078Z","logger":"localqueue-reconciler","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-p7fzd","name":"lq-4ddpz","reconcileID":"0d384946-cf56-4f9b-bc5d-b65d3f2bcda5","error":"Operation cannot be fulfilled on localqueues.kueue.x-k8s.io \"lq-4ddpz\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:33.525616843Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-p7fzd","name":"lq-4ddpz","reconcileID":"8e691ef6-40b7-4c94-9ec8-2fcb36341b89"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:33.806039215Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1276","msg":"Workload create event","replica-role":"leader","workload":{"name":"trainjob-test-runtimepatches-trainjob-kxk7g-bee44","namespace":"test-ns-p7fzd"},"queue":"lq-4ddpz","status":"pending"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:33.806101213Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-p7fzd","name":"trainjob-test-runtimepatches-trainjob-kxk7g-bee44","reconcileID":"d68e69a2-e5e5-48d8-8e2d-2d1269f1e395"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:33.806143809Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-p7fzd","name":"trainjob-test-runtimepatches-trainjob-kxk7g-bee44","reconcileID":"d68e69a2-e5e5-48d8-8e2d-2d1269f1e395","isDeleted":false}
{"level":"Level(-2)","ts":"2026-10-31T13:21:33.806279979Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-p7fzd","name":"trainjob-test-runtimepatches-trainjob-kxk7g-bee44","reconcileID":"7a58a0c4-c566-4e18-8125-97bdbf0d17a9"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:33.806286603Z","logger":"scheduler","caller":"scheduler/scheduler.go:308","msg":"Obtained heads","replica-role":"leader","schedulingCycle":4,"headCount":1,"waitDuration":4.850066229}
{"level":"debug","ts":"2026-10-31T13:21:33.806284418Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"test-runtimepatches-trainjob-kxk7g","namespace":"test-ns-p7fzd"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"CreatedWorkload","action":"CreatedWorkload","note":"Created Workload: test-ns-p7fzd/trainjob-test-runtimepatches-trainjob-kxk7g-bee44"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:33.806322791Z","logger":"scheduler","caller":"scheduler/scheduler.go:323","msg":"Snapshot taken","replica-role":"leader","schedulingCycle":4,"duration":0.000013313}
{"level":"Level(-2)","ts":"2026-10-31T13:21:33.806340225Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"66f7be1d-7e70-4a12-814b-0166cc9f5879","job":"test-ns-qlncb/test-preemption-trainjob-9kdmc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:33.806448142Z","logger":"scheduler","caller":"scheduler/scheduler.go:328","msg":"Nomination done","replica-role":"leader","schedulingCycle":4,"entries":1,"inadmissibleEntries":0,"duration":0.000110712}
{"level":"Level(-2)","ts":"2026-10-31T13:21:33.806477241Z","logger":"scheduler","caller":"scheduler/scheduler.go:406","msg":"Attempting to schedule workload","replica-role":"leader","schedulingCycle":4,"workload":{"name":"trainjob-test-runtimepatches-trainjob-kxk7g-bee44","namespace":"test-ns-p7fzd"},"clusterQueue":{"name":"cq-gd6tf"}}
{"level":"Level(-2)","ts":"2026-10-31T13:21:33.806568675Z","logger":"scheduler","caller":"scheduler/scheduler.go:1041","msg":"Workload assumed in the cache","replica-role":"leader","schedulingCycle":4,"workload":{"name":"trainjob-test-runtimepatches-trainjob-kxk7g-bee44","namespace":"test-ns-p7fzd"},"clusterQueue":{"name":"cq-gd6tf"}}
{"level":"Level(-2)","ts":"2026-10-31T13:21:33.806588946Z","logger":"scheduler","caller":"scheduler/scheduler.go:362","msg":"Workload processing done","replica-role":"leader","schedulingCycle":4,"duration":0.000116194}
{"level":"Level(-2)","ts":"2026-10-31T13:21:33.80659967Z","logger":"scheduler","caller":"scheduler/scheduler.go:297","msg":"Scheduling cycle complete","replica-role":"leader","schedulingCycle":4,"duration":4.850379811}
{"level":"Level(-2)","ts":"2026-10-31T13:21:33.806612815Z","logger":"scheduler","caller":"scheduler/scheduler.go:295","msg":"Scheduling cycle starts","replica-role":"leader","schedulingCycle":5}
{"level":"error","ts":"2026-10-31T13:21:33.858413689Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"66f7be1d-7e70-4a12-814b-0166cc9f5879","job":"test-ns-qlncb/test-preemption-trainjob-9kdmc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-9kdmc\" is forbidden: unable to create new content in namespace test-ns-qlncb because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-10-31T13:21:33.858474633Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"66f7be1d-7e70-4a12-814b-0166cc9f5879","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-9kdmc\" is forbidden: unable to create new content in namespace test-ns-qlncb because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:33.85859749Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-p7fzd","name":"test-runtimepatches-trainjob-kxk7g","reconcileID":"18ad0e07-c6d2-4d5c-9b11-afc5c9a7f2d7","job":"test-ns-p7fzd/test-runtimepatches-trainjob-kxk7g","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:33.916944584Z","logger":"scheduler","caller":"scheduler/scheduler.go:1000","msg":"Workload successfully admitted and assigned flavors","replica-role":"leader","schedulingCycle":4,"workload":{"name":"trainjob-test-runtimepatches-trainjob-kxk7g-bee44","namespace":"test-ns-p7fzd"},"clusterQueue":{"name":"cq-gd6tf"},"assignments":[{"name":"node","flavors":{"cpu":"rf-d5rfn","memory":"rf-d5rfn"},"resourceUsage":{"cpu":"100m","memory":"128Mi"},"count":1}]}
{"level":"debug","ts":"2026-10-31T13:21:33.916945495Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-test-runtimepatches-trainjob-kxk7g-bee44","namespace":"test-ns-p7fzd"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"QuotaReserved","action":"QuotaReserved","note":"Quota reserved in ClusterQueue cq-gd6tf, wait time since queued was 1s"}
{"level":"debug","ts":"2026-10-31T13:21:33.916995923Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-test-runtimepatches-trainjob-kxk7g-bee44","namespace":"test-ns-p7fzd"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"Admitted","action":"Admitted","note":"Admitted by ClusterQueue cq-gd6tf, wait time since reservation was 1s"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:33.917033344Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-p7fzd","name":"trainjob-test-runtimepatches-trainjob-kxk7g-bee44","reconcileID":"ece7ed0b-297f-48cd-a818-f1329a73a625"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:33.917074865Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-p7fzd","name":"trainjob-test-runtimepatches-trainjob-kxk7g-bee44","reconcileID":"ece7ed0b-297f-48cd-a818-f1329a73a625","isDeleted":false}
{"level":"Level(-2)","ts":"2026-10-31T13:21:33.917038478Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-runtimepatches-trainjob-kxk7g-bee44","namespace":"test-ns-p7fzd"},"queue":"lq-4ddpz","status":"admitted","prevStatus":"pending","clusterQueue":"cq-gd6tf"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:33.917340434Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-p7fzd","name":"trainjob-test-runtimepatches-trainjob-kxk7g-bee44","reconcileID":"2ab304c6-d833-438b-8edd-da3227bcef59"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:34.018509115Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-p7fzd","name":"test-runtimepatches-trainjob-kxk7g","reconcileID":"37a67170-7bff-4491-8bce-189fe4db832e","job":"test-ns-p7fzd/test-runtimepatches-trainjob-kxk7g","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:34.178335715Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-p7fzd","name":"test-runtimepatches-trainjob-kxk7g","reconcileID":"37a67170-7bff-4491-8bce-189fe4db832e","job":"test-ns-p7fzd/test-runtimepatches-trainjob-kxk7g","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-3)","ts":"2026-10-31T13:21:34.234118123Z","logger":"trainjob","caller":"jobframework/reconciler.go:594","msg":"Unsuspending job","replica-role":"leader","namespace":"test-ns-p7fzd","name":"test-runtimepatches-trainjob-kxk7g","reconcileID":"37a67170-7bff-4491-8bce-189fe4db832e","job":"test-ns-p7fzd/test-runtimepatches-trainjob-kxk7g","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-runtimepatches-trainjob-kxk7g\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-3)","ts":"2026-10-31T13:21:34.234189922Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-p7fzd","name":"test-runtimepatches-trainjob-kxk7g","reconcileID":"37a67170-7bff-4491-8bce-189fe4db832e","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-runtimepatches-trainjob-kxk7g\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:34.234276165Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"6acdf100-7a21-4d5d-842d-f07ee979f606","job":"test-ns-qlncb/test-preemption-trainjob-9kdmc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:34.266060557Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-qlncb","name":"lq-k88rx","reconcileID":"3f0316f5-6934-4a83-bc8a-12786371e199"}
{"level":"error","ts":"2026-10-31T13:21:34.286577428Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"6acdf100-7a21-4d5d-842d-f07ee979f606","job":"test-ns-qlncb/test-preemption-trainjob-9kdmc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-9kdmc\" is forbidden: unable to create new content in namespace test-ns-qlncb because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-10-31T13:21:34.28663945Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"6acdf100-7a21-4d5d-842d-f07ee979f606","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-9kdmc\" is forbidden: unable to create new content in namespace test-ns-qlncb because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:34.286717253Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-p7fzd","name":"test-runtimepatches-trainjob-kxk7g","reconcileID":"de99f3fd-03bc-4d11-b01f-4a04b25d37ad","job":"test-ns-p7fzd/test-runtimepatches-trainjob-kxk7g","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:34.446948254Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-p7fzd","name":"test-runtimepatches-trainjob-kxk7g","reconcileID":"de99f3fd-03bc-4d11-b01f-4a04b25d37ad","job":"test-ns-p7fzd/test-runtimepatches-trainjob-kxk7g","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"info","ts":"2026-10-31T13:21:34.503787684Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-runtimepatches-trainjob-kxk7g","namespace":"test-ns-p7fzd"},"namespace":"test-ns-p7fzd","name":"test-runtimepatches-trainjob-kxk7g","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"ac05d156-5580-48cf-862c-42bffc70396e","replica-role":"leader"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:34.568054092Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-gd6tf"}}
{"level":"Level(-2)","ts":"2026-10-31T13:21:34.568072938Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-p7fzd","name":"trainjob-test-runtimepatches-trainjob-kxk7g-bee44","reconcileID":"4ba4a265-4fba-4d25-b55b-fc8d5414a186"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:34.5681571Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-gd6tf","reconcileID":"2328c570-6149-4e06-9cd7-59fb46406ebc"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:34.585093015Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:171","msg":"ResourceFlavor update event","replica-role":"leader","resourceFlavor":{"name":"rf-d5rfn"}}
{"level":"Level(-2)","ts":"2026-10-31T13:21:34.585167419Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-d5rfn","reconcileID":"8c2df106-c0db-4060-8f1c-fe2bcf52a513"}
{"level":"info","ts":"2026-10-31T13:21:34.618958368Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-runtimepatches-trainjob-kxk7g","namespace":"test-ns-p7fzd"},"namespace":"test-ns-p7fzd","name":"test-runtimepatches-trainjob-kxk7g","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"93d353c9-1a6b-4b3d-9219-272f74e61d42","replica-role":"leader"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:34.627659Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-gd6tf"}}
{"level":"Level(-2)","ts":"2026-10-31T13:21:34.627734321Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-gd6tf","reconcileID":"6daa1604-3720-481a-9067-0cc1769c70d4"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:34.627732612Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-p7fzd","name":"trainjob-test-runtimepatches-trainjob-kxk7g-bee44","reconcileID":"4eab5a90-01e0-4e36-ad5d-a14489164615"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:34.627729929Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-p7fzd","name":"lq-4ddpz","reconcileID":"97311462-0ce0-4121-b349-f93c372414c3"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:34.676390321Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"d9662ec2-3dac-4cff-a1bd-a85b8423ab3d","job":"test-ns-qlncb/test-preemption-trainjob-9kdmc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"debug","ts":"2026-10-31T13:21:34.676402063Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"test-runtimepatches-trainjob-kxk7g","namespace":"test-ns-p7fzd"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"Started","action":"Started","note":"Admitted by clusterQueue cq-gd6tf"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:34.686068811Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-4ddpz","namespace":"test-ns-p7fzd"}}
{"level":"Level(-2)","ts":"2026-10-31T13:21:34.686167976Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-p7fzd","name":"lq-4ddpz","reconcileID":"bf3a3502-ab02-42cf-97af-5f662203587c"}
{"level":"error","ts":"2026-10-31T13:21:34.728007022Z","logger":"events","caller":"events/event_broadcaster.go:270","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-p7fzd","name":"test-runtimepatches-trainjob-kxk7g.18e39fe7d493d411"},"error":"events.events.k8s.io \"test-runtimepatches-trainjob-kxk7g.18e39fe7d493d411\" is forbidden: unable to create new content in namespace test-ns-p7fzd because it is being terminated","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/workspace/upstream/kueue/src/vendor/k8s.io/client-go/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).attemptRecording\n\t/workspace/upstream/kueue/src/vendor/k8s.io/client-go/tools/events/event_broadcaster.go:211\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).recordToSink.func1\n\t/workspace/upstream/kueue/src/vendor/k8s.io/client-go/tools/events/event_broadcaster.go:200"}
{"level":"error","ts":"2026-10-31T13:21:34.728218729Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"d9662ec2-3dac-4cff-a1bd-a85b8423ab3d","job":"test-ns-qlncb/test-preemption-trainjob-9kdmc","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-9kdmc\" is forbidden: unable to create new content in namespace test-ns-qlncb because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-10-31T13:21:34.728282257Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-qlncb","name":"test-preemption-trainjob-9kdmc","reconcileID":"d9662ec2-3dac-4cff-a1bd-a85b8423ab3d","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-9kdmc\" is forbidden: unable to create new content in namespace test-ns-qlncb because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:34.728377678Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-p7fzd","name":"test-runtimepatches-trainjob-kxk7g","reconcileID":"181c49be-e932-4814-a0a2-992ad6c3ce22","job":"test-ns-p7fzd/test-runtimepatches-trainjob-kxk7g","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-10-31T13:21:34.781126467Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-p7fzd","name":"test-runtimepatches-trainjob-kxk7g","reconcileID":"181c49be-e932-4814-a0a2-992ad6c3ce22","job":"test-ns-p7fzd/test-runtimepatches-trainjob-kxk7g","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-runtimepatches-trainjob-kxk7g\" is forbidden: unable to create new content in namespace test-ns-p7fzd because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-10-31T13:21:34.781198228Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-p7fzd","name":"test-runtimepatches-trainjob-kxk7g","reconcileID":"181c49be-e932-4814-a0a2-992ad6c3ce22","error":"workloads.kueue.x-k8s.io \"test-runtimepatches-trainjob-kxk7g\" is forbidden: unable to create new content in namespace test-ns-p7fzd because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:34.786454967Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-p7fzd","name":"test-runtimepatches-trainjob-kxk7g","reconcileID":"1dba98df-e763-4bc0-b675-d19b03867912","job":"test-ns-p7fzd/test-runtimepatches-trainjob-kxk7g","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-10-31T13:21:34.838931742Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-p7fzd","name":"test-runtimepatches-trainjob-kxk7g","reconcileID":"1dba98df-e763-4bc0-b675-d19b03867912","job":"test-ns-p7fzd/test-runtimepatches-trainjob-kxk7g","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-runtimepatches-trainjob-kxk7g\" is forbidden: unable to create new content in namespace test-ns-p7fzd because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-10-31T13:21:34.838989295Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-p7fzd","name":"test-runtimepatches-trainjob-kxk7g","reconcileID":"1dba98df-e763-4bc0-b675-d19b03867912","error":"workloads.kueue.x-k8s.io \"test-runtimepatches-trainjob-kxk7g\" is forbidden: unable to create new content in namespace test-ns-p7fzd because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:34.849262884Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-p7fzd","name":"test-runtimepatches-trainjob-kxk7g","reconcileID":"00cfbdfd-a25f-4f6f-9ee2-d84edefa64bb","job":"test-ns-p7fzd/test-runtimepatches-trainjob-kxk7g","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-10-31T13:21:34.901921865Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-p7fzd","name":"test-runtimepatches-trainjob-kxk7g","reconcileID":"00cfbdfd-a25f-4f6f-9ee2-d84edefa64bb","job":"test-ns-p7fzd/test-runtimepatches-trainjob-kxk7g","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-runtimepatches-trainjob-kxk7g\" is forbidden: unable to create new content in namespace test-ns-p7fzd because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-10-31T13:21:34.901984219Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-p7fzd","name":"test-runtimepatches-trainjob-kxk7g","reconcileID":"00cfbdfd-a25f-4f6f-9ee2-d84edefa64bb","error":"workloads.kueue.x-k8s.io \"test-runtimepatches-trainjob-kxk7g\" is forbidden: unable to create new content in namespace test-ns-p7fzd because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-10-31T13:21:34.902073848Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-p7fzd","name":"test-runtimepatches-trainjob-kxk7g","reconcileID":"58360641-8463-411e-9c80-21120387ada3","job":"test-ns-p7fzd/test-runtimepatches-trainjob-kxk7g","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}