{"level":"Level(-2)","ts":"2026-11-02T19:03:21.179136343Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-7xkmw","name":"e2e-test-flux","reconcileID":"1f006129-3a9a-460d-b5fb-f4ed6abf6e3b","job":"e2e-7xkmw/e2e-test-flux","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-7xkmw"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:21.298608111Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-7xkmw","name":"e2e-test-flux","reconcileID":"4d3659e3-a07b-45d7-9ba7-673c14d9adeb","job":"e2e-7xkmw/e2e-test-flux","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-7xkmw"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:21.739744971Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-f4vk6","name":"e2e-test","reconcileID":"df575844-853a-41a1-a582-3d4fcf8d07d7","job":"e2e-f4vk6/e2e-test","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-f4vk6"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:21.927303265Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-9prpt","name":"e2e-update-time","reconcileID":"a38a7368-84de-45b9-b784-00df75e0695d","job":"e2e-9prpt/e2e-update-time","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-9prpt"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:22.186572766Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-9prpt","name":"e2e-update-time","reconcileID":"483fbfdb-7da6-474d-aeda-ae9979dde18f","job":"e2e-9prpt/e2e-update-time","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-9prpt"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:22.302082801Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-9prpt","name":"e2e-update-time","reconcileID":"935509a4-89d9-4f39-acda-d1c0cb612186","job":"e2e-9prpt/e2e-update-time","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-9prpt"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:22.395513227Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-9prpt","name":"e2e-update-time","reconcileID":"d0fd1599-6122-4a24-bb66-776f102da6f6","job":"e2e-9prpt/e2e-update-time","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-9prpt"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:22.450228836Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-9prpt","name":"e2e-update-time","reconcileID":"67c6f2a3-3ce7-4672-8d5c-984c1cd3da74","job":"e2e-9prpt/e2e-update-time","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-9prpt"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:22.634284683Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-9prpt","name":"e2e-update-time","reconcileID":"fe4c699a-e976-49eb-9879-a18b2de5d2f4","job":"e2e-9prpt/e2e-update-time","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-9prpt"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:23.087641248Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-9prpt","name":"e2e-update-time","reconcileID":"86d913ee-9ef6-4226-9d0e-18d30d5f8900","job":"e2e-9prpt/e2e-update-time","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-9prpt"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:23.200634393Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-9prpt","name":"e2e-update-time","reconcileID":"816d2b27-35d3-44eb-9df8-aeacd29bb615","job":"e2e-9prpt/e2e-update-time","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-9prpt"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:23.274699261Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-sgn57","name":"e2e-test-managed-by","reconcileID":"36e9bba3-00d4-4658-bf95-4050bfc3b6e2","job":"e2e-sgn57/e2e-test-managed-by","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-sgn57"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:26.952442057Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-7xkmw","name":"e2e-test-flux","reconcileID":"ea4bd16d-c887-4642-971b-272bef3be021","job":"e2e-7xkmw/e2e-test-flux","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-7xkmw"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:38.449329159Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-x8v2k","name":"e2e-deadline-job","reconcileID":"65316b58-9057-477c-8363-f4508d0e92b2","job":"e2e-x8v2k/e2e-deadline-job","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-x8v2k"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:38.680040415Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-x8v2k","name":"e2e-deadline-job","reconcileID":"99d2f5c8-3d13-458d-a6ea-79b79c100548","job":"e2e-x8v2k/e2e-deadline-job","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-x8v2k"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:38.889239548Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-x8v2k","name":"e2e-deadline-job","reconcileID":"d27d0340-449c-469e-80b5-1ab8749b1cb1","job":"e2e-x8v2k/e2e-deadline-job","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-x8v2k"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:38.97752326Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-x8v2k","name":"e2e-deadline-job","reconcileID":"82899d1a-318b-4348-ad12-7bd407064943","job":"e2e-x8v2k/e2e-deadline-job","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-x8v2k"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:39.000988016Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-x8v2k","name":"e2e-deadline-job","reconcileID":"d9ecd7d6-c317-4766-82ed-93c56d8d8364","job":"e2e-x8v2k/e2e-deadline-job","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-x8v2k"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:39.111444152Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-x8v2k","name":"e2e-deadline-job","reconcileID":"141e2225-f6e6-4251-85dd-02a31d585113","job":"e2e-x8v2k/e2e-deadline-job","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-x8v2k"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:42.139821835Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-x8v2k","name":"e2e-deadline-job","reconcileID":"f348d3df-eaf1-423f-8cd7-9751e8ce2423","job":"e2e-x8v2k/e2e-deadline-job","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-x8v2k"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:42.251955921Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-x8v2k","name":"e2e-deadline-job","reconcileID":"704f2a98-49e5-47ec-baa0-3d9d8bd60535","job":"e2e-x8v2k/e2e-deadline-job","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-x8v2k"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:48.107936917Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-x8v2k","name":"e2e-deadline-job","reconcileID":"c9d94f04-f7ef-4da5-bfdd-cc31888ad551","job":"e2e-x8v2k/e2e-deadline-job","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-x8v2k"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:48.168256744Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-x8v2k","name":"e2e-deadline-job","reconcileID":"a7d2106d-1955-410a-ae6f-3a4fd158edee","job":"e2e-x8v2k/e2e-deadline-job","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-x8v2k"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:48.333815318Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-x8v2k","name":"e2e-deadline-job","reconcileID":"9662f42a-78e9-4ea0-b97a-6ffd6496c6f8","job":"e2e-x8v2k/e2e-deadline-job","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-x8v2k"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:48.449420816Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-x8v2k","name":"e2e-deadline-job","reconcileID":"e3495084-7f1c-4eae-b3ad-465eaf978013","job":"e2e-x8v2k/e2e-deadline-job","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-x8v2k"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:48.504640255Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-x8v2k","name":"e2e-deadline-job","reconcileID":"a714d07e-0933-4daa-ba58-4cbe78f135e5","job":"e2e-x8v2k/e2e-deadline-job","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-x8v2k"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:48.666502383Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-rd8dn","name":"e2e-test-runtime-update","reconcileID":"7cbb2ee6-e210-4f36-b3dc-54e114223ef6","job":"e2e-rd8dn/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-rd8dn"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:48.888938072Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-rd8dn","name":"e2e-test-runtime-update","reconcileID":"e10f1973-f5f5-42bf-8267-99dfba25e71c","job":"e2e-rd8dn/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-rd8dn"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:49.049314764Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-rd8dn","name":"e2e-test-runtime-update","reconcileID":"fd414e8f-691b-4af3-93ab-1bd637298b4a","job":"e2e-rd8dn/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-rd8dn"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:49.158342946Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-rd8dn","name":"e2e-test-runtime-update","reconcileID":"3e975b3e-5419-4008-8828-39265f19be38","job":"e2e-rd8dn/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-rd8dn"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:49.18339056Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-rd8dn","name":"e2e-test-runtime-update","reconcileID":"0d2d3ddc-2e02-4ea3-91ad-4d4483434164","job":"e2e-rd8dn/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-rd8dn"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:49.268389748Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-rd8dn","name":"e2e-test-runtime-update","reconcileID":"37757319-61e1-4b1a-821a-c4be7ed1035e","job":"e2e-rd8dn/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-rd8dn"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:49.61599069Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-rd8dn","name":"e2e-test-runtime-update","reconcileID":"6742aff3-9f8d-41f3-a945-89d1d5d50017","job":"e2e-rd8dn/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-rd8dn"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:49.728955946Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-rd8dn","name":"e2e-test-runtime-update","reconcileID":"ee5048e7-526d-4311-b055-d49f8ee92fc5","job":"e2e-rd8dn/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-rd8dn"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:49.838408789Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-rd8dn","name":"e2e-test-runtime-update","reconcileID":"f6a9da86-fd62-4372-afc2-ddea3f57ece7","job":"e2e-rd8dn/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-rd8dn"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:49.843203957Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-rd8dn","name":"e2e-test-runtime-update","reconcileID":"bd42d471-1b68-42f0-9814-0fdb41755e5d","job":"e2e-rd8dn/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-rd8dn"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:49.893566148Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-rd8dn","name":"e2e-test-runtime-update","reconcileID":"a904345a-2a30-45bd-839a-5672759fb829","job":"e2e-rd8dn/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-rd8dn"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:50.066039372Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-rd8dn","name":"e2e-test-runtime-update","reconcileID":"bfc309a0-d37e-4c8d-b2a7-695b79fdbdd6","job":"e2e-rd8dn/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-rd8dn"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:50.387229771Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-rd8dn","name":"e2e-test-runtime-update","reconcileID":"f31f75ca-9d82-4996-942e-ff1f00f410a0","job":"e2e-rd8dn/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-rd8dn"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:50.512882504Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-rd8dn","name":"e2e-test-runtime-update","reconcileID":"c346396f-e89a-4afe-af15-6407fc9a8090","job":"e2e-rd8dn/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-rd8dn"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:50.624891993Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-rd8dn","name":"e2e-test-runtime-update","reconcileID":"794df4c3-68d2-4768-8d3b-7165507aa411","job":"e2e-rd8dn/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-rd8dn"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:50.725366872Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-rd8dn","name":"e2e-test-runtime-update","reconcileID":"9b52129f-cfc9-4777-b079-eaa02aac733c","job":"e2e-rd8dn/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-rd8dn"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:50.7807519Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-rd8dn","name":"e2e-test-runtime-update","reconcileID":"d4a19918-02da-4f8b-8017-8ae42fc444e7","job":"e2e-rd8dn/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-rd8dn"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:50.845421345Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-rd8dn","name":"e2e-test-runtime-update","reconcileID":"90aafdc6-87cc-4bab-9fbc-705fa7e016cf","job":"e2e-rd8dn/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-rd8dn"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:50.854560855Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-rd8dn","name":"e2e-test-runtime-update","reconcileID":"de54073d-8c0e-4a59-9588-7f46826f93f3","job":"e2e-rd8dn/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-rd8dn"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:50.9573513Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-rd8dn","name":"e2e-test-runtime-update","reconcileID":"ea977d28-c448-4e20-957c-c097ad73cc1f","job":"e2e-rd8dn/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-rd8dn"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:51.891506681Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-gj44x","name":"e2e-test-runtime-update","reconcileID":"6f93a824-a60c-4408-829d-a6750aca77a5","job":"e2e-gj44x/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-gj44x"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:52.11461549Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-gj44x","name":"e2e-test-runtime-update","reconcileID":"7c210002-8e6a-49b0-83f1-9f83d46cf4ce","job":"e2e-gj44x/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-gj44x"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:52.274175542Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-gj44x","name":"e2e-test-runtime-update","reconcileID":"1064400c-7dc5-4a96-9e25-79f4cb4ffbed","job":"e2e-gj44x/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-gj44x"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:52.38396312Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-gj44x","name":"e2e-test-runtime-update","reconcileID":"de2aaee2-01a9-4b4e-9439-ce26e37790d2","job":"e2e-gj44x/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-gj44x"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:52.409438627Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-gj44x","name":"e2e-test-runtime-update","reconcileID":"bc9a8110-a12b-4c8b-9ddc-cf71f8b955a4","job":"e2e-gj44x/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-gj44x"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:52.495755736Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-gj44x","name":"e2e-test-runtime-update","reconcileID":"03d6fdd1-cc19-429c-af73-0fdd5ce83518","job":"e2e-gj44x/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-gj44x"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:52.843163361Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-gj44x","name":"e2e-test-runtime-update","reconcileID":"fb97b7a6-0a72-4163-83d1-6acf72345d92","job":"e2e-gj44x/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-gj44x"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:52.956486768Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-gj44x","name":"e2e-test-runtime-update","reconcileID":"91f1b07a-1353-47d8-92ce-81887161ca8a","job":"e2e-gj44x/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-gj44x"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:53.067845096Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-gj44x","name":"e2e-test-runtime-update","reconcileID":"d7b30f47-b5ce-4fe6-a65f-847f66a9f07f","job":"e2e-gj44x/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-gj44x"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:53.068042552Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-gj44x","name":"e2e-test-runtime-update","reconcileID":"07e717ab-e3da-47e1-b295-b3600013cfdb","job":"e2e-gj44x/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-gj44x"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:53.126929995Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-gj44x","name":"e2e-test-runtime-update","reconcileID":"6779eaaf-97d4-4629-944c-1955c11fcd5f","job":"e2e-gj44x/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-gj44x"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:53.347019531Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-gj44x","name":"e2e-test-runtime-update","reconcileID":"74fe9ab1-1725-457a-ae69-86c76a4b24a2","job":"e2e-gj44x/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-gj44x"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:53.617598353Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-gj44x","name":"e2e-test-runtime-update","reconcileID":"a7438ad8-d202-454d-b674-36e95958af2a","job":"e2e-gj44x/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-gj44x"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:53.787496504Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-gj44x","name":"e2e-test-runtime-update","reconcileID":"df8898d3-9c70-4382-84ec-7fa559b85158","job":"e2e-gj44x/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-gj44x"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:53.898633575Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-gj44x","name":"e2e-test-runtime-update","reconcileID":"b2f78908-367f-467f-8031-2ea4e93a70a7","job":"e2e-gj44x/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-gj44x"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:53.952990414Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-gj44x","name":"e2e-test-runtime-update","reconcileID":"0c999e29-6358-4d54-8b0f-a8c51724ea25","job":"e2e-gj44x/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-gj44x"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:54.008275317Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-gj44x","name":"e2e-test-runtime-update","reconcileID":"856d73c3-8d96-4869-85a4-3cf185cf4f79","job":"e2e-gj44x/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-gj44x"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:54.082574478Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-gj44x","name":"e2e-test-runtime-update","reconcileID":"32caf2f7-685e-4d4e-b574-096e04048479","job":"e2e-gj44x/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-gj44x"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:54.117976536Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-gj44x","name":"e2e-test-runtime-update","reconcileID":"60bff55e-826b-4001-9f14-3b9f220d3cc0","job":"e2e-gj44x/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-gj44x"}
{"level":"Level(-2)","ts":"2026-11-02T19:03:54.228771958Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"e2e-gj44x","name":"e2e-test-runtime-update","reconcileID":"58707503-295e-4e94-b41a-a2dda5f20457","job":"e2e-gj44x/e2e-test-runtime-update","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"e2e-gj44x"}
{"level":"Level(-2)","ts":"2026-11-02T19:04:58.357029056Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-9przn","name":"test-dra-trainjob-sh2d9","reconcileID":"523c2f05-5b82-400e-acc5-f794fd607478","job":"test-ns-9przn/test-dra-trainjob-sh2d9","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-9przn"}
{"level":"Level(-2)","ts":"2026-11-02T19:04:58.58903239Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-9przn","name":"test-dra-trainjob-sh2d9","reconcileID":"1df283f1-48d7-4526-b806-82b8a55ae202","job":"test-ns-9przn/test-dra-trainjob-sh2d9","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-9przn"}
{"level":"Level(-2)","ts":"2026-11-02T19:04:58.797814611Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-9przn","name":"test-dra-trainjob-sh2d9","reconcileID":"2540b2a9-5ab8-42d9-bdd6-f08bd2c16077","job":"test-ns-9przn/test-dra-trainjob-sh2d9","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-9przn"}
{"level":"Level(-2)","ts":"2026-11-02T19:04:58.911974272Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-9przn","name":"test-dra-trainjob-sh2d9","reconcileID":"ccb93731-05db-433d-be34-3e97a7e05447","job":"test-ns-9przn/test-dra-trainjob-sh2d9","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-9przn"}
{"level":"Level(-2)","ts":"2026-11-02T19:04:59.006432945Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-9przn","name":"test-dra-trainjob-sh2d9","reconcileID":"c6773763-2787-4afc-9236-8cafd1ad365d","job":"test-ns-9przn/test-dra-trainjob-sh2d9","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-9przn"}
{"level":"Level(-2)","ts":"2026-11-02T19:04:59.117073843Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-9przn","name":"test-dra-trainjob-sh2d9","reconcileID":"77069c80-b935-46d3-90ea-8b149979adc5","job":"test-ns-9przn/test-dra-trainjob-sh2d9","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-9przn"}
{"level":"Level(-2)","ts":"2026-11-02T19:05:03.469791345Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-bvw5k","name":"test-trainjob-n6h9j","reconcileID":"403284c2-1bdf-4d04-86ac-cb864ba20bd6","job":"test-ns-bvw5k/test-trainjob-n6h9j","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-bvw5k"}
{"level":"Level(-2)","ts":"2026-11-02T19:05:03.694684356Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-bvw5k","name":"test-trainjob-n6h9j","reconcileID":"528414da-56ab-4605-9640-177b743fe178","job":"test-ns-bvw5k/test-trainjob-n6h9j","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-bvw5k"}
{"level":"Level(-2)","ts":"2026-11-02T19:05:03.904718942Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-bvw5k","name":"test-trainjob-n6h9j","reconcileID":"ab90ec32-1a21-495d-9cf5-bec8c4e9df7a","job":"test-ns-bvw5k/test-trainjob-n6h9j","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-bvw5k"}
{"level":"Level(-2)","ts":"2026-11-02T19:05:04.007912764Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-bvw5k","name":"test-trainjob-n6h9j","reconcileID":"a4fc75c3-f91b-4a4c-b4e8-0baa0c27a3db","job":"test-ns-bvw5k/test-trainjob-n6h9j","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-bvw5k"}
{"level":"Level(-2)","ts":"2026-11-02T19:05:04.015147806Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-bvw5k","name":"test-trainjob-n6h9j","reconcileID":"38afafea-9151-43e5-b6ea-69c0486fd4ce","job":"test-ns-bvw5k/test-trainjob-n6h9j","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-bvw5k"}
{"level":"Level(-2)","ts":"2026-11-02T19:05:04.128017355Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-bvw5k","name":"test-trainjob-n6h9j","reconcileID":"7ca3160e-d297-4fcc-9b9f-d6172835fb0e","job":"test-ns-bvw5k/test-trainjob-n6h9j","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-bvw5k"}
{"level":"Level(-2)","ts":"2026-11-02T19:05:05.618053276Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-9przn","name":"test-dra-trainjob-sh2d9","reconcileID":"76411db2-5944-45e4-a2c3-26ef97fb00ac","job":"test-ns-9przn/test-dra-trainjob-sh2d9","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-9przn"}
{"level":"Level(-2)","ts":"2026-11-02T19:05:05.731410058Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-9przn","name":"test-dra-trainjob-sh2d9","reconcileID":"7eb235e9-17af-44ac-ba9b-27d108bc399e","job":"test-ns-9przn/test-dra-trainjob-sh2d9","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-9przn"}
{"level":"Level(-2)","ts":"2026-11-02T19:05:06.55916973Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-bvw5k","name":"test-trainjob-n6h9j","reconcileID":"eb6a65b6-5bb1-47a7-923f-a843a27de358","job":"test-ns-bvw5k/test-trainjob-n6h9j","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-bvw5k"}
{"level":"Level(-2)","ts":"2026-11-02T19:05:06.671980482Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-bvw5k","name":"test-trainjob-n6h9j","reconcileID":"e71ad797-8f94-4110-bf1c-463bae33cd40","job":"test-ns-bvw5k/test-trainjob-n6h9j","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-bvw5k"}
{"level":"Level(-2)","ts":"2026-11-02T19:05:09.887159083Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-9przn","name":"test-dra-trainjob-sh2d9","reconcileID":"61243953-bdd2-4937-88dc-d656eef218a6","job":"test-ns-9przn/test-dra-trainjob-sh2d9","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-9przn"}
{"level":"Level(-2)","ts":"2026-11-02T19:05:13.598386732Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-bvw5k","name":"test-trainjob-n6h9j","reconcileID":"4d8db258-a633-4a83-b589-db1744dad816","job":"test-ns-bvw5k/test-trainjob-n6h9j","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-bvw5k"}
{"level":"Level(-2)","ts":"2026-11-02T19:05:13.708919763Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-bvw5k","name":"test-trainjob-n6h9j","reconcileID":"238f4d5f-4768-4faf-ab18-81a9ed3b0839","job":"test-ns-bvw5k/test-trainjob-n6h9j","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-bvw5k"}
{"level":"Level(-2)","ts":"2026-11-02T19:05:14.606570845Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-bvw5k","name":"test-trainjob-n6h9j","reconcileID":"790503d3-ca76-47a7-baf2-39b21bc9b396","job":"test-ns-bvw5k/test-trainjob-n6h9j","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-bvw5k"}
{"level":"Level(-2)","ts":"2026-11-02T19:05:14.665164136Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-bvw5k","name":"test-trainjob-n6h9j","reconcileID":"b29c507c-2eac-4031-bdf6-8582c4239e5d","job":"test-ns-bvw5k/test-trainjob-n6h9j","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-bvw5k"}
{"level":"Level(-2)","ts":"2026-11-02T19:05:14.719067759Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-bvw5k","name":"test-trainjob-n6h9j","reconcileID":"070590c7-4ea0-49f5-b43e-eb866a95aa8b","job":"test-ns-bvw5k/test-trainjob-n6h9j","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-bvw5k"}
{"level":"Level(-2)","ts":"2026-11-02T19:05:14.830190045Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-bvw5k","name":"test-trainjob-n6h9j","reconcileID":"bae7d410-9207-4ede-bdbb-d7ec6edbe150","job":"test-ns-bvw5k/test-trainjob-n6h9j","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-bvw5k"}
{"level":"Level(-2)","ts":"2026-11-02T19:05:15.838796331Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-lxz44","name":"test-trainjob-xgvkj","reconcileID":"321a097d-ce01-40a3-b80e-3e5aa649b406","job":"test-ns-lxz44/test-trainjob-xgvkj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-lxz44"}
{"level":"Level(-2)","ts":"2026-11-02T19:05:16.07276055Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-lxz44","name":"test-trainjob-xgvkj","reconcileID":"2bb92f93-a47a-44b2-8aa3-783549f097a1","job":"test-ns-lxz44/test-trainjob-xgvkj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-lxz44"}
{"level":"Level(-2)","ts":"2026-11-02T19:05:16.275015913Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-lxz44","name":"test-trainjob-xgvkj","reconcileID":"965999f8-8cd1-4be4-be63-0381de5836a0","job":"test-ns-lxz44/test-trainjob-xgvkj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-lxz44"}
{"level":"Level(-2)","ts":"2026-11-02T19:05:16.383390475Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-lxz44","name":"test-trainjob-xgvkj","reconcileID":"0f306a0e-cae6-47a4-a1da-d9e0f75b1951","job":"test-ns-lxz44/test-trainjob-xgvkj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-lxz44"}
{"level":"Level(-2)","ts":"2026-11-02T19:05:16.384697845Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-lxz44","name":"test-trainjob-xgvkj","reconcileID":"5aaf6877-6c17-44e8-9054-e48918dc751d","job":"test-ns-lxz44/test-trainjob-xgvkj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-lxz44"}
{"level":"Level(-2)","ts":"2026-11-02T19:05:16.494886146Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-lxz44","name":"test-trainjob-xgvkj","reconcileID":"fa052188-0991-4ee5-9434-069790d8631c","job":"test-ns-lxz44/test-trainjob-xgvkj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-lxz44"}
{"level":"Level(-2)","ts":"2026-11-02T19:16:35.075206521Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-lxz44","name":"test-trainjob-xgvkj","reconcileID":"a6b9d470-d8cb-45f9-ad36-178cda81414e","job":"test-ns-lxz44/test-trainjob-xgvkj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-lxz44"}
{"level":"Level(-2)","ts":"2026-11-02T19:16:35.189067379Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-lxz44","name":"test-trainjob-xgvkj","reconcileID":"8315a4aa-988d-4481-ad23-868668c8ce42","job":"test-ns-lxz44/test-trainjob-xgvkj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-lxz44"}
{"level":"Level(-2)","ts":"2026-11-02T19:16:40.039503945Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-lxz44","name":"test-trainjob-xgvkj","reconcileID":"4fe9e5ff-5f9c-49ff-8ffb-5d444fdc43da","job":"test-ns-lxz44/test-trainjob-xgvkj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-lxz44"}
{"level":"Level(-2)","ts":"2026-11-02T19:16:40.152401158Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-lxz44","name":"test-trainjob-xgvkj","reconcileID":"574e9b2f-2c59-4972-9122-0159879645fd","job":"test-ns-lxz44/test-trainjob-xgvkj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-lxz44"}
{"level":"Level(-2)","ts":"2026-11-02T19:16:41.047966429Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-lxz44","name":"test-trainjob-xgvkj","reconcileID":"6b008b0b-09d3-4c8c-b892-e0592a764ccb","job":"test-ns-lxz44/test-trainjob-xgvkj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-lxz44"}
{"level":"Level(-2)","ts":"2026-11-02T19:16:41.156260169Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-lxz44","name":"test-trainjob-xgvkj","reconcileID":"e5af105c-feca-408b-87b7-5dc6c0923ed5","job":"test-ns-lxz44/test-trainjob-xgvkj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-lxz44"}
{"level":"Level(-2)","ts":"2026-11-02T19:16:41.158564334Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-lxz44","name":"test-trainjob-xgvkj","reconcileID":"3b8200fb-5f08-4662-b4ee-1d599f268e11","job":"test-ns-lxz44/test-trainjob-xgvkj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-lxz44"}
{"level":"Level(-2)","ts":"2026-11-02T19:16:41.269273842Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-lxz44","name":"test-trainjob-xgvkj","reconcileID":"96ffcdf5-8b3d-4ba7-a29d-1469dc018ada","job":"test-ns-lxz44/test-trainjob-xgvkj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-lxz44"}
{"level":"Level(-2)","ts":"2026-11-02T19:16:42.192892726Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-8hbhx","name":"test-trainjob-j9qf4","reconcileID":"ccff7281-d3ba-4d61-8efe-d6a4ec257511","job":"test-ns-8hbhx/test-trainjob-j9qf4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-8hbhx"}
{"level":"Level(-2)","ts":"2026-11-02T19:16:42.675896466Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-8hbhx","name":"test-trainjob-j9qf4","reconcileID":"605627a2-b62f-4e8f-acb2-37a977a488b4","job":"test-ns-8hbhx/test-trainjob-j9qf4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-8hbhx"}
{"level":"Level(-2)","ts":"2026-11-02T19:16:42.885467655Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-8hbhx","name":"test-trainjob-j9qf4","reconcileID":"77a547a3-6867-423f-8e96-3ee8532f2458","job":"test-ns-8hbhx/test-trainjob-j9qf4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-8hbhx"}
{"level":"Level(-2)","ts":"2026-11-02T19:16:42.996198356Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-8hbhx","name":"test-trainjob-j9qf4","reconcileID":"bddc5cd0-0b8d-401c-9b3b-8645ac3ae846","job":"test-ns-8hbhx/test-trainjob-j9qf4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-8hbhx"}
{"level":"Level(-2)","ts":"2026-11-02T19:16:43.251515473Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-8hbhx","name":"test-trainjob-j9qf4","reconcileID":"63fc923e-5d6d-4fd4-8ca5-2d5a8d5d8bc2","job":"test-ns-8hbhx/test-trainjob-j9qf4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-8hbhx"}
{"level":"Level(-2)","ts":"2026-11-02T19:16:43.362714506Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-8hbhx","name":"test-trainjob-j9qf4","reconcileID":"d7b5616d-cf8d-42cd-95f0-ac45d699fea2","job":"test-ns-8hbhx/test-trainjob-j9qf4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-8hbhx"}
{"level":"Level(-2)","ts":"2026-11-02T19:20:10.130939388Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-8hbhx","name":"test-trainjob-j9qf4","reconcileID":"d1b6c030-a24f-4e5c-9996-2cc127fce71f","job":"test-ns-8hbhx/test-trainjob-j9qf4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-8hbhx"}
{"level":"Level(-2)","ts":"2026-11-02T19:20:10.241999409Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-8hbhx","name":"test-trainjob-j9qf4","reconcileID":"bfe34ea6-6024-485d-9f75-7598d0a78088","job":"test-ns-8hbhx/test-trainjob-j9qf4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-8hbhx"}
{"level":"Level(-2)","ts":"2026-11-02T19:20:14.135573041Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-8hbhx","name":"test-trainjob-j9qf4","reconcileID":"536eb1f4-056a-4092-aa32-93e79861c0a6","job":"test-ns-8hbhx/test-trainjob-j9qf4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-8hbhx"}
{"level":"Level(-2)","ts":"2026-11-02T19:20:14.248311081Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-8hbhx","name":"test-trainjob-j9qf4","reconcileID":"d5e345df-aa5e-47b0-a748-d36fea4bc0c8","job":"test-ns-8hbhx/test-trainjob-j9qf4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-8hbhx"}
{"level":"Level(-2)","ts":"2026-11-02T19:20:15.142432103Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-8hbhx","name":"test-trainjob-j9qf4","reconcileID":"8458c668-d7d6-43d1-bebe-74cf50bc1433","job":"test-ns-8hbhx/test-trainjob-j9qf4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-8hbhx"}
{"level":"Level(-2)","ts":"2026-11-02T19:20:15.238071592Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-8hbhx","name":"test-trainjob-j9qf4","reconcileID":"0ceffa29-f3b6-4d3e-b88a-848c4a0d0916","job":"test-ns-8hbhx/test-trainjob-j9qf4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-8hbhx"}
{"level":"Level(-2)","ts":"2026-11-02T19:20:15.255140985Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-8hbhx","name":"test-trainjob-j9qf4","reconcileID":"d846d1fa-b2a2-4c42-a773-025ca8683b9f","job":"test-ns-8hbhx/test-trainjob-j9qf4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-8hbhx"}
{"level":"Level(-2)","ts":"2026-11-02T19:20:15.367657777Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-8hbhx","name":"test-trainjob-j9qf4","reconcileID":"07e06fee-6010-46f2-8477-977b1e3f07a8","job":"test-ns-8hbhx/test-trainjob-j9qf4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-8hbhx"}
{"level":"Level(-2)","ts":"2026-11-02T19:20:17.304286671Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-lfv2l","name":"test-trainjob-kp9jg","reconcileID":"1a184eeb-92ea-4d74-b9cc-352eb46b6d8b","job":"test-ns-lfv2l/test-trainjob-kp9jg","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-lfv2l"}
{"level":"Level(-2)","ts":"2026-11-02T19:20:17.834544977Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-lfv2l","name":"test-trainjob-kp9jg","reconcileID":"99c31521-5688-47cb-884a-ac00a6cccd9e","job":"test-ns-lfv2l/test-trainjob-kp9jg","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-lfv2l"}
{"level":"Level(-2)","ts":"2026-11-02T19:20:18.048601093Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-lfv2l","name":"test-trainjob-kp9jg","reconcileID":"3e190fb7-7bcd-4603-9770-dac3b53fd8f5","job":"test-ns-lfv2l/test-trainjob-kp9jg","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-lfv2l"}
{"level":"Level(-2)","ts":"2026-11-02T19:20:18.163486199Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-lfv2l","name":"test-trainjob-kp9jg","reconcileID":"02c15f4d-c661-43eb-b30d-bfda1366ee1e","job":"test-ns-lfv2l/test-trainjob-kp9jg","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-lfv2l"}
{"level":"Level(-2)","ts":"2026-11-02T19:20:18.373141461Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-lfv2l","name":"test-trainjob-kp9jg","reconcileID":"3ceb1ae6-05e9-47aa-9103-f87a034a828a","job":"test-ns-lfv2l/test-trainjob-kp9jg","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-lfv2l"}
{"level":"Level(-2)","ts":"2026-11-02T19:20:18.497339442Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-lfv2l","name":"test-trainjob-kp9jg","reconcileID":"e4e4713f-f247-4144-a5b7-901e55715cc8","job":"test-ns-lfv2l/test-trainjob-kp9jg","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-lfv2l"}
{"level":"Level(-2)","ts":"2026-11-02T19:20:21.706925744Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-8hbhx","name":"test-trainjob-j9qf4","reconcileID":"c5aef473-5a83-4955-ac2d-9740c2470993","job":"test-ns-8hbhx/test-trainjob-j9qf4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-8hbhx"}
{"level":"Level(-2)","ts":"2026-11-02T19:20:21.81665838Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-8hbhx","name":"test-trainjob-j9qf4","reconcileID":"b2354044-ea4e-4ee3-baa3-1a0cd0eb3949","job":"test-ns-8hbhx/test-trainjob-j9qf4","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-8hbhx"}
{"level":"Level(-2)","ts":"2026-11-02T19:20:22.144364459Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-lxz44","name":"test-trainjob-xgvkj","reconcileID":"a8fc517d-6f6e-4329-bdc8-ac68eaf61477","job":"test-ns-lxz44/test-trainjob-xgvkj","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-lxz44"}
{"level":"Level(-2)","ts":"2026-11-02T19:20:41.275953394Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-lfv2l","name":"test-trainjob-kp9jg","reconcileID":"d15434ab-2dad-430d-ba79-53aa6fba4cea","job":"test-ns-lfv2l/test-trainjob-kp9jg","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-lfv2l"}
{"level":"Level(-2)","ts":"2026-11-02T19:20:41.391531632Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-lfv2l","name":"test-trainjob-kp9jg","reconcileID":"992df952-752c-41ae-86e7-b768fec8d572","job":"test-ns-lfv2l/test-trainjob-kp9jg","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-lfv2l"}
{"level":"Level(-2)","ts":"2026-11-02T19:20:46.303140005Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-lfv2l","name":"test-trainjob-kp9jg","reconcileID":"10c385c5-e38c-4d05-8480-4cdb908ca9b4","job":"test-ns-lfv2l/test-trainjob-kp9jg","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-lfv2l"}
{"level":"Level(-2)","ts":"2026-11-02T19:20:46.419706079Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-lfv2l","name":"test-trainjob-kp9jg","reconcileID":"5d71478a-2238-4057-848e-7c703ec9bc57","job":"test-ns-lfv2l/test-trainjob-kp9jg","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-lfv2l"}
{"level":"Level(-2)","ts":"2026-11-02T19:20:47.310962766Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-lfv2l","name":"test-trainjob-kp9jg","reconcileID":"8604bfb2-5684-47fd-9e38-6c58631158e0","job":"test-ns-lfv2l/test-trainjob-kp9jg","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-lfv2l"}
{"level":"Level(-2)","ts":"2026-11-02T19:20:47.426574288Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-lfv2l","name":"test-trainjob-kp9jg","reconcileID":"8ce728b7-97f7-4de3-acb4-cbf84b0baff7","job":"test-ns-lfv2l/test-trainjob-kp9jg","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-lfv2l"}
{"level":"Level(-2)","ts":"2026-11-02T19:20:47.524288522Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-lfv2l","name":"test-trainjob-kp9jg","reconcileID":"6928f90f-f5e0-450f-9830-2d2986a114f7","job":"test-ns-lfv2l/test-trainjob-kp9jg","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-lfv2l"}
{"level":"Level(-2)","ts":"2026-11-02T19:20:47.640858467Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-lfv2l","name":"test-trainjob-kp9jg","reconcileID":"5c679b3f-541c-466f-a35f-c63cb90ffbe9","job":"test-ns-lfv2l/test-trainjob-kp9jg","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-lfv2l"}
{"level":"Level(-2)","ts":"2026-11-02T19:20:47.641533317Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-lfv2l","name":"test-trainjob-kp9jg","reconcileID":"d698c40f-c165-44b1-a8d7-1f33c42df49a","job":"test-ns-lfv2l/test-trainjob-kp9jg","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-lfv2l"}
{"level":"Level(-2)","ts":"2026-11-02T19:20:47.757004828Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-lfv2l","name":"test-trainjob-kp9jg","reconcileID":"0118dccc-6936-4e5a-9637-af90820ab713","job":"test-ns-lfv2l/test-trainjob-kp9jg","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-lfv2l"}
{"level":"Level(-2)","ts":"2026-11-02T19:24:00.276006022Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-lfv2l","name":"test-trainjob-kp9jg","reconcileID":"16bf9fe6-9972-4b11-92e6-cf559ac9d1aa","job":"test-ns-lfv2l/test-trainjob-kp9jg","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-lfv2l"}
{"level":"Level(-2)","ts":"2026-11-02T19:24:00.393446392Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-lfv2l","name":"test-trainjob-kp9jg","reconcileID":"d1591af5-a7da-4d77-9951-b5fd0167935d","job":"test-ns-lfv2l/test-trainjob-kp9jg","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-lfv2l"}
{"level":"Level(-2)","ts":"2026-11-02T19:24:11.316267438Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-lfv2l","name":"test-trainjob-kp9jg","reconcileID":"3f7b44e0-8762-4c11-ad46-ca7bc5fb8b29","job":"test-ns-lfv2l/test-trainjob-kp9jg","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-lfv2l"}
{"level":"Level(-2)","ts":"2026-11-02T19:24:11.435365736Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-lfv2l","name":"test-trainjob-kp9jg","reconcileID":"e4f3e7af-0499-4c71-8882-a6e854e8192e","job":"test-ns-lfv2l/test-trainjob-kp9jg","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-lfv2l"}
{"level":"Level(-2)","ts":"2026-11-02T19:24:12.324829852Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-lfv2l","name":"test-trainjob-kp9jg","reconcileID":"7b49c715-b638-4677-b24a-a3cd188f4848","job":"test-ns-lfv2l/test-trainjob-kp9jg","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-lfv2l"}
{"level":"Level(-2)","ts":"2026-11-02T19:24:12.441171882Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-lfv2l","name":"test-trainjob-kp9jg","reconcileID":"020274c9-8065-432f-ad5e-733b4f514e71","job":"test-ns-lfv2l/test-trainjob-kp9jg","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-lfv2l"}
{"level":"Level(-2)","ts":"2026-11-02T19:24:12.536731952Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-lfv2l","name":"test-trainjob-kp9jg","reconcileID":"9a68b7eb-d479-408b-bd7e-04ade3c18976","job":"test-ns-lfv2l/test-trainjob-kp9jg","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-lfv2l"}
{"level":"Level(-2)","ts":"2026-11-02T19:24:12.651454033Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-lfv2l","name":"test-trainjob-kp9jg","reconcileID":"ea218418-9dd0-445c-8654-e4fc5bddd73c","job":"test-ns-lfv2l/test-trainjob-kp9jg","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-lfv2l"}
{"level":"Level(-2)","ts":"2026-11-02T19:24:12.866304981Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-lfv2l","name":"test-trainjob-kp9jg","reconcileID":"22bdcca9-24a9-4053-a71f-2cac188c56e3","job":"test-ns-lfv2l/test-trainjob-kp9jg","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-lfv2l"}
{"level":"Level(-2)","ts":"2026-11-02T19:24:12.982805765Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-lfv2l","name":"test-trainjob-kp9jg","reconcileID":"521f1763-d178-4afd-94c0-12e6dc25d0a9","job":"test-ns-lfv2l/test-trainjob-kp9jg","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-lfv2l"}
{"level":"Level(-2)","ts":"2026-11-02T19:24:20.517032451Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-lfv2l","name":"test-trainjob-kp9jg","reconcileID":"467b5886-c53c-47a1-8b9e-6c88d37b08bf","job":"test-ns-lfv2l/test-trainjob-kp9jg","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-lfv2l"}
{"level":"Level(-2)","ts":"2026-11-02T19:24:20.633021936Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-lfv2l","name":"test-trainjob-kp9jg","reconcileID":"f486fb57-61e6-4cab-bd0b-5830817e1018","job":"test-ns-lfv2l/test-trainjob-kp9jg","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-lfv2l"}
{"level":"Level(-2)","ts":"2026-11-02T19:24:20.666006586Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-lfv2l","name":"test-trainjob-kp9jg","reconcileID":"19c67261-b5b3-4303-ad36-3116afd794d3","job":"test-ns-lfv2l/test-trainjob-kp9jg","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-lfv2l"}
{"level":"Level(-2)","ts":"2026-11-02T19:24:20.749205689Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-lfv2l","name":"test-trainjob-kp9jg","reconcileID":"82ebb822-5f6d-4ccb-a52e-661852ef6129","job":"test-ns-lfv2l/test-trainjob-kp9jg","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-lfv2l"}
{"level":"Level(-2)","ts":"2026-11-02T19:24:42.749777238Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-rqxl8","name":"s59ff52c38ac","reconcileID":"e59d006e-513b-43ba-9a67-fd48b392608e","job":"test-ns-rqxl8/s59ff52c38ac","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-rqxl8"}
{"level":"Level(-2)","ts":"2026-11-02T19:24:43.26210322Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-rqxl8","name":"s59ff52c38ac","reconcileID":"9bb04bc8-6a6c-4478-b97b-5c6349c7aa78","job":"test-ns-rqxl8/s59ff52c38ac","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-rqxl8"}
{"level":"Level(-2)","ts":"2026-11-02T19:24:43.473547925Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-rqxl8","name":"s59ff52c38ac","reconcileID":"8e195180-c907-4862-b57c-9976099ead5b","job":"test-ns-rqxl8/s59ff52c38ac","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-rqxl8"}
{"level":"Level(-2)","ts":"2026-11-02T19:24:43.588483349Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-rqxl8","name":"s59ff52c38ac","reconcileID":"4696bdf5-9b53-4023-b2e8-5d30360d96b4","job":"test-ns-rqxl8/s59ff52c38ac","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-rqxl8"}
{"level":"Level(-2)","ts":"2026-11-02T19:24:43.728568845Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-rqxl8","name":"s59ff52c38ac","reconcileID":"d8489897-4c54-441b-bafd-b0ca1901b6d1","job":"test-ns-rqxl8/s59ff52c38ac","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-rqxl8"}
{"level":"Level(-2)","ts":"2026-11-02T19:24:43.845812871Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-rqxl8","name":"s59ff52c38ac","reconcileID":"2961f077-0302-4358-9a99-a7c214f0f666","job":"test-ns-rqxl8/s59ff52c38ac","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-rqxl8"}
{"level":"Level(-2)","ts":"2026-11-02T19:24:45.672690767Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-rqxl8","name":"s59ff52c38ac","reconcileID":"d9b5b406-c0d9-4373-a973-e3ddc8f49040","job":"test-ns-rqxl8/s59ff52c38ac","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-rqxl8"}
{"level":"Level(-2)","ts":"2026-11-02T19:24:45.789242613Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-rqxl8","name":"s59ff52c38ac","reconcileID":"790cdc9c-db42-402e-b819-87311be12f8c","job":"test-ns-rqxl8/s59ff52c38ac","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-rqxl8"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:19.776921674Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-rqxl8","name":"s59ff52c38ac","reconcileID":"1a2f0934-d684-4a0a-94f3-4a9e6b48ad46","job":"test-ns-rqxl8/s59ff52c38ac","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-rqxl8"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:19.892581449Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-rqxl8","name":"s59ff52c38ac","reconcileID":"216e1f71-3d21-4889-8003-40ea189d0745","job":"test-ns-rqxl8/s59ff52c38ac","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-rqxl8"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:20.797207242Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-rqxl8","name":"s59ff52c38ac","reconcileID":"ba44477b-8cfb-4b94-be19-6306b197777a","job":"test-ns-rqxl8/s59ff52c38ac","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-rqxl8"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:20.879230806Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-rqxl8","name":"s59ff52c38ac","reconcileID":"4c9c1366-ae7f-4996-8d99-b0b2d00c9206","job":"test-ns-rqxl8/s59ff52c38ac","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-rqxl8"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:20.913342736Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-rqxl8","name":"s59ff52c38ac","reconcileID":"5003feca-f5c1-478a-ae88-f005f611446c","job":"test-ns-rqxl8/s59ff52c38ac","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-rqxl8"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:21.045553976Z","logger":"trainjob","caller":"jobframework/reconciler.go:326","msg":"Namespace not opted in for Kueue management","replica-role":"leader","namespace":"test-ns-rqxl8","name":"s59ff52c38ac","reconcileID":"5528c3db-69b5-4fac-b377-fed3918fd7d5","job":"test-ns-rqxl8/s59ff52c38ac","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","namespace":"test-ns-rqxl8"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:25.400513402Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:141","msg":"ResourceFlavor create event","replica-role":"leader","resourceFlavor":{"name":"rf-smvtt"}}
{"level":"Level(-2)","ts":"2026-11-02T19:25:25.400657308Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-smvtt","reconcileID":"75644d5c-9904-4f26-83c8-1eb88e21800b"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:25.766949742Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:351","msg":"ClusterQueue create event","replica-role":"leader","clusterQueue":{"name":"cq-jrsgx"}}
{"level":"Level(-2)","ts":"2026-11-02T19:25:25.767260279Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-jrsgx","reconcileID":"e86771c0-1395-4347-abc6-a749b6a257e8"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:25.784471759Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:240","msg":"LocalQueue create event","replica-role":"leader","localQueue":{"name":"lq-6qbzx","namespace":"test-ns-gws47"}}
{"level":"Level(-2)","ts":"2026-11-02T19:25:25.784643029Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-gws47","name":"lq-6qbzx","reconcileID":"3ea01175-280c-4f27-ba9c-0a4af6ba2c99"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:25.822534678Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-jrsgx"}}
{"level":"Level(-2)","ts":"2026-11-02T19:25:25.822743118Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-jrsgx","reconcileID":"cfae15ff-e966-4e64-87bc-d0248b315fbd"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:25.83854425Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-gws47","name":"lq-6qbzx","reconcileID":"8c1d9780-2dbe-4c25-9723-05cbc37b9887"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:25.83856331Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-6qbzx","namespace":"test-ns-gws47"}}
{"level":"Level(-2)","ts":"2026-11-02T19:25:25.893688476Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-6qbzx","namespace":"test-ns-gws47"}}
{"level":"Level(-2)","ts":"2026-11-02T19:25:25.893705923Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-gws47","name":"lq-6qbzx","reconcileID":"3b5b8032-ed24-4780-8a58-e0ba555ea2c5"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:25.893847359Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-gws47","name":"lq-6qbzx","reconcileID":"f2f8487a-7d5f-438c-b2bd-615cb328e762"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:26.652724614Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gws47","name":"dw-deployment-ppjlp-8556fdc7bd-lcfgh","reconcileID":"1bc86e5d-a4ef-4db2-b549-fc6ae868bca5","job":"test-ns-gws47/dw-deployment-ppjlp-8556fdc7bd-lcfgh","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:26.701420745Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1.PriorityClass","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"debug","ts":"2026-11-02T19:25:27.151639019Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"dw-deployment-ppjlp-8556fdc7bd-lcfgh","namespace":"test-ns-gws47"},"kind":"Pod","apiVersion":"v1","type":"Normal","reason":"CreatedWorkload","action":"CreatedWorkload","note":"Created Workload: test-ns-gws47/pod-dw-deployment-ppjlp-8556fdc7bd-lcfgh-ea040"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:27.198066533Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1276","msg":"Workload create event","replica-role":"leader","workload":{"name":"pod-dw-deployment-ppjlp-8556fdc7bd-lcfgh-ea040","namespace":"test-ns-gws47"},"queue":"lq-6qbzx","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:27.198090321Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-gws47","name":"pod-dw-deployment-ppjlp-8556fdc7bd-lcfgh-ea040","reconcileID":"42ddd566-1ea5-4ad1-8ab0-068259cae348"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:27.198154772Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-gws47","name":"pod-dw-deployment-ppjlp-8556fdc7bd-lcfgh-ea040","reconcileID":"42ddd566-1ea5-4ad1-8ab0-068259cae348","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T19:25:27.198153391Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gws47","name":"dw-deployment-ppjlp-8556fdc7bd-lcfgh","reconcileID":"e9edd0f4-3804-4860-9e6d-47b02ebd15ac","job":"test-ns-gws47/dw-deployment-ppjlp-8556fdc7bd-lcfgh","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:27.198319227Z","logger":"scheduler","caller":"scheduler/scheduler.go:308","msg":"Obtained heads","replica-role":"leader","schedulingCycle":1,"headCount":1,"waitDuration":2310.512615904}
{"level":"Level(-2)","ts":"2026-11-02T19:25:27.198355782Z","logger":"scheduler","caller":"scheduler/scheduler.go:323","msg":"Snapshot taken","replica-role":"leader","schedulingCycle":1,"duration":0.000010041}
{"level":"Level(-2)","ts":"2026-11-02T19:25:27.198387557Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-gws47","name":"pod-dw-deployment-ppjlp-8556fdc7bd-lcfgh-ea040","reconcileID":"00bdced0-96b6-463a-be2f-b9a831c15902"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:27.198453551Z","logger":"scheduler","caller":"scheduler/scheduler.go:328","msg":"Nomination done","replica-role":"leader","schedulingCycle":1,"entries":1,"inadmissibleEntries":0,"duration":0.000088376}
{"level":"Level(-2)","ts":"2026-11-02T19:25:27.198475143Z","logger":"scheduler","caller":"scheduler/scheduler.go:406","msg":"Attempting to schedule workload","replica-role":"leader","schedulingCycle":1,"workload":{"name":"pod-dw-deployment-ppjlp-8556fdc7bd-lcfgh-ea040","namespace":"test-ns-gws47"},"clusterQueue":{"name":"cq-jrsgx"}}
{"level":"Level(-2)","ts":"2026-11-02T19:25:27.198561228Z","logger":"scheduler","caller":"scheduler/scheduler.go:1041","msg":"Workload assumed in the cache","replica-role":"leader","schedulingCycle":1,"workload":{"name":"pod-dw-deployment-ppjlp-8556fdc7bd-lcfgh-ea040","namespace":"test-ns-gws47"},"clusterQueue":{"name":"cq-jrsgx"}}
{"level":"Level(-2)","ts":"2026-11-02T19:25:27.198595476Z","logger":"scheduler","caller":"scheduler/scheduler.go:362","msg":"Workload processing done","replica-role":"leader","schedulingCycle":1,"duration":0.000125298}
{"level":"Level(-2)","ts":"2026-11-02T19:25:27.198611729Z","logger":"scheduler","caller":"scheduler/scheduler.go:297","msg":"Scheduling cycle complete","replica-role":"leader","schedulingCycle":1,"duration":2310.512909573}
{"level":"Level(-2)","ts":"2026-11-02T19:25:27.198626222Z","logger":"scheduler","caller":"scheduler/scheduler.go:295","msg":"Scheduling cycle starts","replica-role":"leader","schedulingCycle":2}
{"level":"Level(-2)","ts":"2026-11-02T19:25:27.312022748Z","logger":"scheduler","caller":"scheduler/scheduler.go:1000","msg":"Workload successfully admitted and assigned flavors","replica-role":"leader","schedulingCycle":1,"workload":{"name":"pod-dw-deployment-ppjlp-8556fdc7bd-lcfgh-ea040","namespace":"test-ns-gws47"},"clusterQueue":{"name":"cq-jrsgx"},"assignments":[{"name":"main","flavors":{"cpu":"rf-smvtt","memory":"rf-smvtt"},"resourceUsage":{"cpu":"1","memory":"3Gi"},"count":1}]}
{"level":"debug","ts":"2026-11-02T19:25:27.31206988Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"pod-dw-deployment-ppjlp-8556fdc7bd-lcfgh-ea040","namespace":"test-ns-gws47"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"QuotaReserved","action":"QuotaReserved","note":"Quota reserved in ClusterQueue cq-jrsgx, wait time since queued was 1s"}
{"level":"debug","ts":"2026-11-02T19:25:27.312103729Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"pod-dw-deployment-ppjlp-8556fdc7bd-lcfgh-ea040","namespace":"test-ns-gws47"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"Admitted","action":"Admitted","note":"Admitted by ClusterQueue cq-jrsgx, wait time since reservation was 0s"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:27.312581301Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-gws47","name":"pod-dw-deployment-ppjlp-8556fdc7bd-lcfgh-ea040","reconcileID":"303a4631-d731-4c24-a4e9-4d8ea2d69e9f"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:27.312598724Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"pod-dw-deployment-ppjlp-8556fdc7bd-lcfgh-ea040","namespace":"test-ns-gws47"},"queue":"lq-6qbzx","status":"admitted","prevStatus":"pending","clusterQueue":"cq-jrsgx"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:27.312638247Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-gws47","name":"pod-dw-deployment-ppjlp-8556fdc7bd-lcfgh-ea040","reconcileID":"303a4631-d731-4c24-a4e9-4d8ea2d69e9f","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T19:25:27.312663517Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gws47","name":"dw-deployment-ppjlp-8556fdc7bd-lcfgh","reconcileID":"d162672e-be65-4969-b1a8-aec01cae07db","job":"test-ns-gws47/dw-deployment-ppjlp-8556fdc7bd-lcfgh","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:27.312875633Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-gws47","name":"pod-dw-deployment-ppjlp-8556fdc7bd-lcfgh-ea040","reconcileID":"ef6200f6-bc97-46c0-991f-123ae4f2aba0"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:27.312894974Z","logger":"v1_pod","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-gws47","name":"dw-deployment-ppjlp-8556fdc7bd-lcfgh","reconcileID":"d162672e-be65-4969-b1a8-aec01cae07db","job":"test-ns-gws47/dw-deployment-ppjlp-8556fdc7bd-lcfgh","gvk":"/v1, Kind=Pod"}
{"level":"debug","ts":"2026-11-02T19:25:27.428007443Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"dw-deployment-ppjlp-8556fdc7bd-lcfgh","namespace":"test-ns-gws47"},"kind":"Pod","apiVersion":"v1","type":"Normal","reason":"Started","action":"Started","note":"Admitted by clusterQueue cq-jrsgx"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:27.428040138Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gws47","name":"dw-deployment-ppjlp-8556fdc7bd-lcfgh","reconcileID":"e19c0a1b-84a6-4ac6-8575-250d84034798","job":"test-ns-gws47/dw-deployment-ppjlp-8556fdc7bd-lcfgh","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:27.435236213Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gws47","name":"dw-deployment-ppjlp-8556fdc7bd-lcfgh","reconcileID":"ece2f8d9-2e8b-4cac-9d34-685583706a47","job":"test-ns-gws47/dw-deployment-ppjlp-8556fdc7bd-lcfgh","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:27.494915747Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gws47","name":"dw-deployment-ppjlp-8556fdc7bd-lcfgh","reconcileID":"87d57efe-f276-4610-b71a-d485fa389faf","job":"test-ns-gws47/dw-deployment-ppjlp-8556fdc7bd-lcfgh","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:27.60893612Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gws47","name":"dw-deployment-ppjlp-8556fdc7bd-lcfgh","reconcileID":"1cfbd3c2-f0a5-460d-8741-58c5a95bddf0","job":"test-ns-gws47/dw-deployment-ppjlp-8556fdc7bd-lcfgh","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:28.198779579Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-gws47","name":"lq-6qbzx","reconcileID":"784dca0a-e1a5-40a7-8d15-3d796b3d9f64"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:28.19880394Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-jrsgx","reconcileID":"3c5f4c09-0cd1-4a8c-ba21-dd0b95c4d4bc"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:28.253324383Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-6qbzx","namespace":"test-ns-gws47"}}
{"level":"Level(-2)","ts":"2026-11-02T19:25:28.253435226Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-gws47","name":"lq-6qbzx","reconcileID":"8b1b2a11-2beb-46cb-b568-60ed11d42ec1"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:28.299805603Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-jrsgx"}}
{"level":"Level(-2)","ts":"2026-11-02T19:25:28.299970017Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-jrsgx","reconcileID":"1af14c7d-f210-415b-aab1-7bc3dc18ff97"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:28.428306912Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-gws47","name":"pod-dw-deployment-ppjlp-8556fdc7bd-lcfgh-ea040","reconcileID":"049538c7-a7a4-4661-b046-e3c6c711b21c"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:28.440105769Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gws47","name":"dw-deployment-ppjlp-8556fdc7bd-lcfgh","reconcileID":"4972482e-85d0-4b49-ab47-64dc05c6de23","job":"test-ns-gws47/dw-deployment-ppjlp-8556fdc7bd-lcfgh","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:29.441059544Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-gws47","name":"pod-dw-deployment-ppjlp-8556fdc7bd-lcfgh-ea040","reconcileID":"21cc56ef-63b0-4842-86de-601d8e2bd585"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:29.720129021Z","logger":"v1_pod","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gws47","name":"dw-deployment-ppjlp-8556fdc7bd-lcfgh","reconcileID":"c236c190-1d82-43f6-a1a9-b4939ad8fa14","job":"test-ns-gws47/dw-deployment-ppjlp-8556fdc7bd-lcfgh","gvk":"/v1, Kind=Pod"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:30.721094106Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-gws47","name":"pod-dw-deployment-ppjlp-8556fdc7bd-lcfgh-ea040","reconcileID":"6b3caa23-e0a5-49d0-9690-2f31972fa653"}
{"level":"info","ts":"2026-11-02T19:25:43.719898938Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:117","msg":"Validating create","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"q42f4ce24259","namespace":"test-ns-gws47"},"namespace":"test-ns-gws47","name":"q42f4ce24259","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:opendatahub:trainer-e2e","requestID":"6e0725bd-6b44-4f98-95ed-4e0b3159cfac","replica-role":"leader"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:43.782494119Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gws47","name":"q42f4ce24259","reconcileID":"7d97138d-6f26-4789-8e1a-d8e54e238e81","job":"test-ns-gws47/q42f4ce24259","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:44.100039425Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gws47","name":"q42f4ce24259","reconcileID":"2db4627b-b54a-4f8b-842d-9b7ead75bfe1","job":"test-ns-gws47/q42f4ce24259","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"debug","ts":"2026-11-02T19:25:44.100027485Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"q42f4ce24259","namespace":"test-ns-gws47"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"CreatedWorkload","action":"CreatedWorkload","note":"Created Workload: test-ns-gws47/trainjob-q42f4ce24259-ec4c6"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:44.100365904Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1276","msg":"Workload create event","replica-role":"leader","workload":{"name":"trainjob-q42f4ce24259-ec4c6","namespace":"test-ns-gws47"},"queue":"lq-6qbzx","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:44.100360562Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-gws47","name":"trainjob-q42f4ce24259-ec4c6","reconcileID":"4c369a34-a3a2-454b-abbe-ee20449336a6"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:44.100480271Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-gws47","name":"trainjob-q42f4ce24259-ec4c6","reconcileID":"4c369a34-a3a2-454b-abbe-ee20449336a6","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T19:25:44.100564565Z","logger":"scheduler","caller":"scheduler/scheduler.go:308","msg":"Obtained heads","replica-role":"leader","schedulingCycle":2,"headCount":1,"waitDuration":16.901937351}
{"level":"Level(-2)","ts":"2026-11-02T19:25:44.10061662Z","logger":"scheduler","caller":"scheduler/scheduler.go:323","msg":"Snapshot taken","replica-role":"leader","schedulingCycle":2,"duration":0.000013205}
{"level":"Level(-2)","ts":"2026-11-02T19:25:44.100615266Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-gws47","name":"trainjob-q42f4ce24259-ec4c6","reconcileID":"d043430c-bb93-426a-b712-de69386d3df6"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:44.100707788Z","logger":"scheduler","caller":"scheduler/scheduler.go:328","msg":"Nomination done","replica-role":"leader","schedulingCycle":2,"entries":1,"inadmissibleEntries":0,"duration":0.000078397}
{"level":"Level(-2)","ts":"2026-11-02T19:25:44.10072368Z","logger":"scheduler","caller":"scheduler/scheduler.go:406","msg":"Attempting to schedule workload","replica-role":"leader","schedulingCycle":2,"workload":{"name":"trainjob-q42f4ce24259-ec4c6","namespace":"test-ns-gws47"},"clusterQueue":{"name":"cq-jrsgx"}}
{"level":"Level(-2)","ts":"2026-11-02T19:25:44.100815881Z","logger":"scheduler","caller":"scheduler/scheduler.go:1041","msg":"Workload assumed in the cache","replica-role":"leader","schedulingCycle":2,"workload":{"name":"trainjob-q42f4ce24259-ec4c6","namespace":"test-ns-gws47"},"clusterQueue":{"name":"cq-jrsgx"}}
{"level":"Level(-2)","ts":"2026-11-02T19:25:44.100847139Z","logger":"scheduler","caller":"scheduler/scheduler.go:362","msg":"Workload processing done","replica-role":"leader","schedulingCycle":2,"duration":0.00012665}
{"level":"Level(-2)","ts":"2026-11-02T19:25:44.100865619Z","logger":"scheduler","caller":"scheduler/scheduler.go:297","msg":"Scheduling cycle complete","replica-role":"leader","schedulingCycle":2,"duration":16.902239001}
{"level":"Level(-2)","ts":"2026-11-02T19:25:44.100883355Z","logger":"scheduler","caller":"scheduler/scheduler.go:295","msg":"Scheduling cycle starts","replica-role":"leader","schedulingCycle":3}
{"level":"Level(-2)","ts":"2026-11-02T19:25:44.210473061Z","logger":"scheduler","caller":"scheduler/scheduler.go:1000","msg":"Workload successfully admitted and assigned flavors","replica-role":"leader","schedulingCycle":2,"workload":{"name":"trainjob-q42f4ce24259-ec4c6","namespace":"test-ns-gws47"},"clusterQueue":{"name":"cq-jrsgx"},"assignments":[{"name":"node","flavors":{"cpu":"rf-smvtt","memory":"rf-smvtt"},"resourceUsage":{"cpu":"4","memory":"16Gi"},"count":2}]}
{"level":"debug","ts":"2026-11-02T19:25:44.210546229Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-q42f4ce24259-ec4c6","namespace":"test-ns-gws47"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"QuotaReserved","action":"QuotaReserved","note":"Quota reserved in ClusterQueue cq-jrsgx, wait time since queued was 0s"}
{"level":"debug","ts":"2026-11-02T19:25:44.210629755Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-q42f4ce24259-ec4c6","namespace":"test-ns-gws47"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"Admitted","action":"Admitted","note":"Admitted by ClusterQueue cq-jrsgx, wait time since reservation was 0s"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:44.210948283Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-q42f4ce24259-ec4c6","namespace":"test-ns-gws47"},"queue":"lq-6qbzx","status":"admitted","prevStatus":"pending","clusterQueue":"cq-jrsgx"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:44.210966933Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-gws47","name":"trainjob-q42f4ce24259-ec4c6","reconcileID":"1e164fa2-f11b-44d7-b6b9-528707f59cf6"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:44.211023958Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-gws47","name":"trainjob-q42f4ce24259-ec4c6","reconcileID":"1e164fa2-f11b-44d7-b6b9-528707f59cf6","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T19:25:44.211194004Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-gws47","name":"trainjob-q42f4ce24259-ec4c6","reconcileID":"b1d13fb7-47bd-4932-8f91-4cac816a4821"}
{"level":"error","ts":"2026-11-02T19:25:44.421357484Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-gws47","name":"q42f4ce24259","reconcileID":"2db4627b-b54a-4f8b-842d-9b7ead75bfe1","error":"workloads.kueue.x-k8s.io \"trainjob-q42f4ce24259-ec4c6\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:44.421485957Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gws47","name":"q42f4ce24259","reconcileID":"edfdaa22-f381-4257-98bc-efc096a7d014","job":"test-ns-gws47/q42f4ce24259","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:44.578852984Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-gws47","name":"q42f4ce24259","reconcileID":"edfdaa22-f381-4257-98bc-efc096a7d014","job":"test-ns-gws47/q42f4ce24259","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"info","ts":"2026-11-02T19:25:44.694644804Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"q42f4ce24259","namespace":"test-ns-gws47"},"namespace":"test-ns-gws47","name":"q42f4ce24259","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"968c0858-2a8f-4ffd-ba2d-1958710f6926","replica-role":"leader"}
{"level":"info","ts":"2026-11-02T19:25:44.863294928Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"q42f4ce24259","namespace":"test-ns-gws47"},"namespace":"test-ns-gws47","name":"q42f4ce24259","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"abf99b4a-aecf-460a-b55c-479f0ebf215b","replica-role":"leader"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:44.916880186Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gws47","name":"q42f4ce24259","reconcileID":"18180e17-bcd6-4945-867e-2cb4ac9a09dc","job":"test-ns-gws47/q42f4ce24259","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"debug","ts":"2026-11-02T19:25:44.916889966Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"q42f4ce24259","namespace":"test-ns-gws47"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"Started","action":"Started","note":"Admitted by clusterQueue cq-jrsgx"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:45.074184016Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-gws47","name":"q42f4ce24259","reconcileID":"18180e17-bcd6-4945-867e-2cb4ac9a09dc","job":"test-ns-gws47/q42f4ce24259","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:45.100991746Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-gws47","name":"lq-6qbzx","reconcileID":"2b4366fb-7e30-4437-ba0b-034daf8bc126"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:45.100994981Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-jrsgx","reconcileID":"5f950287-5e97-44e4-8225-1c2d29d45ae5"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:45.155522588Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-6qbzx","namespace":"test-ns-gws47"}}
{"level":"Level(-2)","ts":"2026-11-02T19:25:45.155661283Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-gws47","name":"lq-6qbzx","reconcileID":"40ab6d76-1e0a-4e89-81e1-24578de0689b"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:45.156709788Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-jrsgx"}}
{"level":"Level(-2)","ts":"2026-11-02T19:25:45.156866088Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-jrsgx","reconcileID":"d2e20376-9c9c-48d5-8f03-d3fe57fcfd63"}
{"level":"Level(-3)","ts":"2026-11-02T19:25:45.181545644Z","logger":"trainjob","caller":"jobframework/reconciler.go:594","msg":"Unsuspending job","replica-role":"leader","namespace":"test-ns-gws47","name":"q42f4ce24259","reconcileID":"18180e17-bcd6-4945-867e-2cb4ac9a09dc","job":"test-ns-gws47/q42f4ce24259","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"q42f4ce24259\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-3)","ts":"2026-11-02T19:25:45.181625839Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-gws47","name":"q42f4ce24259","reconcileID":"18180e17-bcd6-4945-867e-2cb4ac9a09dc","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"q42f4ce24259\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:45.181746715Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gws47","name":"q42f4ce24259","reconcileID":"fea66aa2-58ee-4d91-a5a8-190b12198a8f","job":"test-ns-gws47/q42f4ce24259","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:45.339585369Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gws47","name":"q42f4ce24259","reconcileID":"3c8376fb-0ebb-4eea-99a4-dc4bdc2eecd8","job":"test-ns-gws47/q42f4ce24259","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:45.497649192Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gws47","name":"q42f4ce24259","reconcileID":"7c58ffdf-810c-4f78-96cf-feae60123cb0","job":"test-ns-gws47/q42f4ce24259","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:45.968321289Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gws47","name":"q42f4ce24259","reconcileID":"2e3168ad-c4b7-4672-a090-d7c66f829d96","job":"test-ns-gws47/q42f4ce24259","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:46.126135414Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gws47","name":"q42f4ce24259","reconcileID":"be3dc510-6378-4e79-8d1f-e858b29b7a28","job":"test-ns-gws47/q42f4ce24259","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:46.687379946Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-gws47","name":"trainjob-q42f4ce24259-ec4c6","reconcileID":"c59f3ae6-994d-4106-afba-d02f58ff7b6b"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:47.795144498Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-gws47","name":"trainjob-q42f4ce24259-ec4c6","reconcileID":"2f7a6f2c-04b4-4ec4-95d4-d5910b3c867a"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:48.909469135Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-gws47","name":"trainjob-q42f4ce24259-ec4c6","reconcileID":"88441380-89d6-4ae4-958a-5c86cc8ade41"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:48.931154686Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gws47","name":"q42f4ce24259","reconcileID":"5f1e7c39-6d4d-447f-86dd-050cadb564ef","job":"test-ns-gws47/q42f4ce24259","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T19:25:49.089005152Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gws47","name":"q42f4ce24259","reconcileID":"6adf3125-68db-47f4-81b6-90b884b362cd","job":"test-ns-gws47/q42f4ce24259","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:22.172652425Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-gws47","name":"trainjob-q42f4ce24259-ec4c6","reconcileID":"a5632a85-1662-44bc-9b50-3c5f9308a0db"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:22.195025226Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gws47","name":"q42f4ce24259","reconcileID":"9c993bc1-b58e-4f56-803a-3216125abdf6","job":"test-ns-gws47/q42f4ce24259","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:22.358902002Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gws47","name":"q42f4ce24259","reconcileID":"4844e856-e1d9-4e3c-b9ba-0efb68060d29","job":"test-ns-gws47/q42f4ce24259","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:23.204805814Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gws47","name":"q42f4ce24259","reconcileID":"4d55f268-02ce-4417-8b10-69985406301b","job":"test-ns-gws47/q42f4ce24259","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:23.278415236Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-gws47","name":"trainjob-q42f4ce24259-ec4c6","reconcileID":"8fe35276-74ef-4b9e-a0db-d4928ef8d80a"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:23.363476521Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gws47","name":"q42f4ce24259","reconcileID":"2282f4f6-21fe-4acc-9c07-e24d8fac6e68","job":"test-ns-gws47/q42f4ce24259","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:23.521339694Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gws47","name":"q42f4ce24259","reconcileID":"6c2c2572-df10-4ce9-a4b4-d1991894dec5","job":"test-ns-gws47/q42f4ce24259","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"debug","ts":"2026-11-02T19:26:23.791250401Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"q42f4ce24259","namespace":"test-ns-gws47"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"FinishedWorkload","action":"FinishedWorkload","note":"Workload 'test-ns-gws47/trainjob-q42f4ce24259-ec4c6' is declared finished"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:23.791514008Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-q42f4ce24259-ec4c6","namespace":"test-ns-gws47"},"queue":"lq-6qbzx","status":"finished","prevStatus":"admitted","clusterQueue":"cq-jrsgx"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:23.791574758Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-gws47","name":"trainjob-q42f4ce24259-ec4c6","reconcileID":"6523906e-3732-4c12-ad79-16ab6bb043ee"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:23.791633089Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-gws47","name":"trainjob-q42f4ce24259-ec4c6","reconcileID":"6523906e-3732-4c12-ad79-16ab6bb043ee","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T19:26:23.79169648Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gws47","name":"q42f4ce24259","reconcileID":"bbe02a49-f3f7-421a-a72b-16349d8a7f77","job":"test-ns-gws47/q42f4ce24259","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:23.791715926Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-gws47","name":"trainjob-q42f4ce24259-ec4c6","reconcileID":"5d602596-b233-44a7-8281-adef6e01ff73"}
{"level":"debug","ts":"2026-11-02T19:26:23.949008623Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"q42f4ce24259","namespace":"test-ns-gws47"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"FinishedWorkload","action":"FinishedWorkload","note":"Workload 'test-ns-gws47/trainjob-q42f4ce24259-ec4c6' is declared finished"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:24.060826639Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-q42f4ce24259-ec4c6","namespace":"test-ns-gws47"},"queue":"lq-6qbzx","status":"finished","clusterQueue":"cq-jrsgx"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:24.060853731Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-gws47","name":"trainjob-q42f4ce24259-ec4c6","reconcileID":"8322a618-14d3-44e8-abda-1c9fc41c6c76"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:24.060924735Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-gws47","name":"trainjob-q42f4ce24259-ec4c6","reconcileID":"8322a618-14d3-44e8-abda-1c9fc41c6c76","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T19:26:24.060955727Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-gws47","name":"q42f4ce24259","reconcileID":"6e519148-76b6-4a0a-815b-3ba24666d48e","job":"test-ns-gws47/q42f4ce24259","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:24.061013267Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-gws47","name":"trainjob-q42f4ce24259-ec4c6","reconcileID":"1b39207d-2a83-411a-a4b7-4448dcc3ef31"}
{"level":"debug","ts":"2026-11-02T19:26:24.222324048Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"q42f4ce24259","namespace":"test-ns-gws47"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"FinishedWorkload","action":"FinishedWorkload","note":"Workload 'test-ns-gws47/trainjob-q42f4ce24259-ec4c6' is declared finished"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:24.792785399Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-jrsgx","reconcileID":"b718de11-a704-4c8c-afe2-da3cc67cf811"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:24.792788437Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-gws47","name":"lq-6qbzx","reconcileID":"c7fa9030-fcc6-4b3b-b253-e0d0728373b7"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:24.845885064Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-6qbzx","namespace":"test-ns-gws47"}}
{"level":"Level(-2)","ts":"2026-11-02T19:26:24.84600415Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-gws47","name":"lq-6qbzx","reconcileID":"b42e064f-97cf-4a35-8517-1aea1c2760b1"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:24.848811687Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-jrsgx"}}
{"level":"Level(-2)","ts":"2026-11-02T19:26:24.84896422Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-jrsgx","reconcileID":"e8273f5b-2631-457b-8b35-7f4b306211bb"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:26.90823647Z","logger":"trainjob","caller":"workload/workload.go:1810","msg":"Workload is orphaned; finishing to release quota","replica-role":"leader","namespace":"test-ns-gws47","name":"q42f4ce24259","reconcileID":"8f13b182-6da7-47c9-8cc1-a573a950c50d","job":"test-ns-gws47/q42f4ce24259","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:26.954488568Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1312","msg":"Workload delete event","replica-role":"leader","workload":{"name":"trainjob-q42f4ce24259-ec4c6","namespace":"test-ns-gws47"},"queue":"lq-6qbzx","status":"finished"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:26.954545758Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-gws47","name":"trainjob-q42f4ce24259-ec4c6","reconcileID":"bc437755-7d38-4766-a4ce-871b4ba19538"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:26.954595563Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-gws47","name":"trainjob-q42f4ce24259-ec4c6","reconcileID":"bc437755-7d38-4766-a4ce-871b4ba19538","isDeleted":true}
{"level":"Level(-2)","ts":"2026-11-02T19:26:27.955223214Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-gws47","name":"lq-6qbzx","reconcileID":"618b60cd-c35a-4829-8eb7-039d76f90474"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:27.955262987Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-jrsgx","reconcileID":"51da683e-2c48-4701-aaa1-0ddd70b3f301"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:28.497863804Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-jrsgx"}}
{"level":"Level(-2)","ts":"2026-11-02T19:26:28.497977898Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-jrsgx","reconcileID":"978a0b44-a9f0-4944-accb-6a248bd238d3"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:28.498070712Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-gws47","name":"pod-dw-deployment-ppjlp-8556fdc7bd-lcfgh-ea040","reconcileID":"5fe0c232-24fd-45fb-9fa0-795147742c01"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:28.514257955Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:171","msg":"ResourceFlavor update event","replica-role":"leader","resourceFlavor":{"name":"rf-smvtt"}}
{"level":"Level(-2)","ts":"2026-11-02T19:26:28.514380178Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-smvtt","reconcileID":"8585b61b-f575-4fc8-a75c-bb892794c1a5"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:28.553032351Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-jrsgx"}}
{"level":"Level(-2)","ts":"2026-11-02T19:26:28.553149409Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-gws47","name":"lq-6qbzx","reconcileID":"e6b430df-ecd5-4415-9d9f-3af57f50b9c8"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:28.553154972Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-jrsgx","reconcileID":"b83ca517-84bf-4c21-9838-e86d2806325f"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:28.553230759Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-gws47","name":"pod-dw-deployment-ppjlp-8556fdc7bd-lcfgh-ea040","reconcileID":"6adb5a73-6e10-4925-bbb7-067ac3ac5bcc"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:28.607406329Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-6qbzx","namespace":"test-ns-gws47"}}
{"level":"Level(-2)","ts":"2026-11-02T19:26:28.607511394Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-gws47","name":"lq-6qbzx","reconcileID":"5479a989-685e-4efe-bba2-51a0bd762386"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:28.8568692Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:141","msg":"ResourceFlavor create event","replica-role":"leader","resourceFlavor":{"name":"rf-nkprn"}}
{"level":"Level(-2)","ts":"2026-11-02T19:26:28.856973589Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-nkprn","reconcileID":"2ab5b80e-a1f1-4eec-a5fa-9e1cd6c15d41"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:28.97932633Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:351","msg":"ClusterQueue create event","replica-role":"leader","clusterQueue":{"name":"cq-rf2xr"}}
{"level":"Level(-2)","ts":"2026-11-02T19:26:28.979530832Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-rf2xr","reconcileID":"b634a7cd-dd74-490b-84e3-c72f042851ce"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:28.997733166Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:240","msg":"LocalQueue create event","replica-role":"leader","localQueue":{"name":"lq-6dt7b","namespace":"test-ns-z95z9"}}
{"level":"Level(-2)","ts":"2026-11-02T19:26:28.997888704Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-z95z9","name":"lq-6dt7b","reconcileID":"b8a51304-724d-4a80-8206-c344a5647a99"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:29.032548806Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-rf2xr"}}
{"level":"Level(-2)","ts":"2026-11-02T19:26:29.032720995Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-rf2xr","reconcileID":"22838518-2271-4d06-b664-6072f05316fd"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:29.052204346Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-6dt7b","namespace":"test-ns-z95z9"}}
{"level":"Level(-2)","ts":"2026-11-02T19:26:29.052213263Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-z95z9","name":"lq-6dt7b","reconcileID":"f0ea7670-96f4-43a1-9d99-809e1a5e7673"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:29.106977529Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-6dt7b","namespace":"test-ns-z95z9"}}
{"level":"Level(-2)","ts":"2026-11-02T19:26:29.106985069Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-z95z9","name":"lq-6dt7b","reconcileID":"39216e49-a34d-4652-99ce-5055bd80fc4f"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:29.107184595Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-z95z9","name":"lq-6dt7b","reconcileID":"326ce644-fe1b-4754-8f3e-cb16748862bb"}
{"level":"info","ts":"2026-11-02T19:26:29.123098098Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:117","msg":"Validating create","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-preemption-trainjob-h67sh","namespace":"test-ns-z95z9"},"namespace":"test-ns-z95z9","name":"test-preemption-trainjob-h67sh","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:admin","requestID":"2f0d40aa-31e9-468c-8b20-b13c17d3413d","replica-role":"leader"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:29.175085929Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-z95z9","name":"test-preemption-trainjob-h67sh","reconcileID":"119c891a-4afa-4d53-b95d-189559c57bd7","job":"test-ns-z95z9/test-preemption-trainjob-h67sh","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:29.479749059Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-gws47","name":"pod-dw-deployment-ppjlp-8556fdc7bd-lcfgh-ea040","reconcileID":"74657f89-f11c-40e0-bc03-dafdbde65a7a"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:29.488641872Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1276","msg":"Workload create event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-h67sh-61b2a","namespace":"test-ns-z95z9"},"queue":"lq-6dt7b","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:29.488649944Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-z95z9","name":"trainjob-test-preemption-trainjob-h67sh-61b2a","reconcileID":"ee610cc1-1ba3-41b8-9585-cb2dd753d10c"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:29.488872831Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-z95z9","name":"trainjob-test-preemption-trainjob-h67sh-61b2a","reconcileID":"ee610cc1-1ba3-41b8-9585-cb2dd753d10c","isDeleted":false}
{"level":"debug","ts":"2026-11-02T19:26:29.488683039Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"test-preemption-trainjob-h67sh","namespace":"test-ns-z95z9"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"CreatedWorkload","action":"CreatedWorkload","note":"Created Workload: test-ns-z95z9/trainjob-test-preemption-trainjob-h67sh-61b2a"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:29.488718455Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-z95z9","name":"test-preemption-trainjob-h67sh","reconcileID":"a4d90640-5746-480c-a62e-4515a10ae3fd","job":"test-ns-z95z9/test-preemption-trainjob-h67sh","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:29.488886727Z","logger":"scheduler","caller":"scheduler/scheduler.go:308","msg":"Obtained heads","replica-role":"leader","schedulingCycle":3,"headCount":1,"waitDuration":45.388002809}
{"level":"Level(-2)","ts":"2026-11-02T19:26:29.488914833Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-z95z9","name":"trainjob-test-preemption-trainjob-h67sh-61b2a","reconcileID":"90519c58-2deb-42d5-a68f-39e17207ad79"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:29.488932204Z","logger":"scheduler","caller":"scheduler/scheduler.go:323","msg":"Snapshot taken","replica-role":"leader","schedulingCycle":3,"duration":0.000010849}
{"level":"Level(-2)","ts":"2026-11-02T19:26:29.488996446Z","logger":"scheduler","caller":"scheduler/scheduler.go:328","msg":"Nomination done","replica-role":"leader","schedulingCycle":3,"entries":1,"inadmissibleEntries":0,"duration":0.000056375}
{"level":"Level(-2)","ts":"2026-11-02T19:26:29.489014221Z","logger":"scheduler","caller":"scheduler/scheduler.go:406","msg":"Attempting to schedule workload","replica-role":"leader","schedulingCycle":3,"workload":{"name":"trainjob-test-preemption-trainjob-h67sh-61b2a","namespace":"test-ns-z95z9"},"clusterQueue":{"name":"cq-rf2xr"}}
{"level":"Level(-2)","ts":"2026-11-02T19:26:29.489074206Z","logger":"scheduler","caller":"scheduler/scheduler.go:1041","msg":"Workload assumed in the cache","replica-role":"leader","schedulingCycle":3,"workload":{"name":"trainjob-test-preemption-trainjob-h67sh-61b2a","namespace":"test-ns-z95z9"},"clusterQueue":{"name":"cq-rf2xr"}}
{"level":"Level(-2)","ts":"2026-11-02T19:26:29.489092587Z","logger":"scheduler","caller":"scheduler/scheduler.go:362","msg":"Workload processing done","replica-role":"leader","schedulingCycle":3,"duration":0.00008213}
{"level":"Level(-2)","ts":"2026-11-02T19:26:29.489101838Z","logger":"scheduler","caller":"scheduler/scheduler.go:297","msg":"Scheduling cycle complete","replica-role":"leader","schedulingCycle":3,"duration":45.388218358}
{"level":"Level(-2)","ts":"2026-11-02T19:26:29.489140519Z","logger":"scheduler","caller":"scheduler/scheduler.go:295","msg":"Scheduling cycle starts","replica-role":"leader","schedulingCycle":4}
{"level":"Level(-2)","ts":"2026-11-02T19:26:29.596709832Z","logger":"scheduler","caller":"scheduler/scheduler.go:1000","msg":"Workload successfully admitted and assigned flavors","replica-role":"leader","schedulingCycle":3,"workload":{"name":"trainjob-test-preemption-trainjob-h67sh-61b2a","namespace":"test-ns-z95z9"},"clusterQueue":{"name":"cq-rf2xr"},"assignments":[{"name":"node","count":1}]}
{"level":"Level(-2)","ts":"2026-11-02T19:26:29.596747805Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-z95z9","name":"trainjob-test-preemption-trainjob-h67sh-61b2a","reconcileID":"589d709d-a924-4cae-8ceb-6abcbd7c168a"}
{"level":"debug","ts":"2026-11-02T19:26:29.596781402Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-test-preemption-trainjob-h67sh-61b2a","namespace":"test-ns-z95z9"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"QuotaReserved","action":"QuotaReserved","note":"Quota reserved in ClusterQueue cq-rf2xr, wait time since queued was 1s"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:29.596804934Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-z95z9","name":"trainjob-test-preemption-trainjob-h67sh-61b2a","reconcileID":"589d709d-a924-4cae-8ceb-6abcbd7c168a","isDeleted":false}
{"level":"debug","ts":"2026-11-02T19:26:29.596808073Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-test-preemption-trainjob-h67sh-61b2a","namespace":"test-ns-z95z9"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"Admitted","action":"Admitted","note":"Admitted by ClusterQueue cq-rf2xr, wait time since reservation was 1s"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:29.596799561Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-h67sh-61b2a","namespace":"test-ns-z95z9"},"queue":"lq-6dt7b","status":"admitted","prevStatus":"pending","clusterQueue":"cq-rf2xr"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:29.596987256Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-z95z9","name":"trainjob-test-preemption-trainjob-h67sh-61b2a","reconcileID":"861332f9-b526-44e8-ba35-5162b600d3fb"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:29.642824308Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-z95z9","name":"test-preemption-trainjob-h67sh","reconcileID":"a0eb5a2f-624c-4860-b855-b35a18f881f3","job":"test-ns-z95z9/test-preemption-trainjob-h67sh","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:29.797040081Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-z95z9","name":"test-preemption-trainjob-h67sh","reconcileID":"a0eb5a2f-624c-4860-b855-b35a18f881f3","job":"test-ns-z95z9/test-preemption-trainjob-h67sh","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-3)","ts":"2026-11-02T19:26:29.902511276Z","logger":"trainjob","caller":"jobframework/reconciler.go:594","msg":"Unsuspending job","replica-role":"leader","namespace":"test-ns-z95z9","name":"test-preemption-trainjob-h67sh","reconcileID":"a0eb5a2f-624c-4860-b855-b35a18f881f3","job":"test-ns-z95z9/test-preemption-trainjob-h67sh","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-preemption-trainjob-h67sh\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-3)","ts":"2026-11-02T19:26:29.90258987Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-z95z9","name":"test-preemption-trainjob-h67sh","reconcileID":"a0eb5a2f-624c-4860-b855-b35a18f881f3","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-preemption-trainjob-h67sh\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:29.902688835Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-z95z9","name":"test-preemption-trainjob-h67sh","reconcileID":"af1ae223-dcca-4fd0-95eb-10e5b566e1e7","job":"test-ns-z95z9/test-preemption-trainjob-h67sh","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:30.057597872Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-z95z9","name":"test-preemption-trainjob-h67sh","reconcileID":"af1ae223-dcca-4fd0-95eb-10e5b566e1e7","job":"test-ns-z95z9/test-preemption-trainjob-h67sh","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"info","ts":"2026-11-02T19:26:30.167754958Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-preemption-trainjob-h67sh","namespace":"test-ns-z95z9"},"namespace":"test-ns-z95z9","name":"test-preemption-trainjob-h67sh","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"2b72de62-153c-4d08-9234-f0ccaf4e2f41","replica-role":"leader"}
{"level":"info","ts":"2026-11-02T19:26:30.331714868Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-preemption-trainjob-h67sh","namespace":"test-ns-z95z9"},"namespace":"test-ns-z95z9","name":"test-preemption-trainjob-h67sh","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"46fbd39d-4716-42d4-ac09-648f3399ef36","replica-role":"leader"}
{"level":"debug","ts":"2026-11-02T19:26:30.384927796Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"test-preemption-trainjob-h67sh","namespace":"test-ns-z95z9"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"Started","action":"Started","note":"Admitted by clusterQueue cq-rf2xr"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:30.384960558Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-z95z9","name":"test-preemption-trainjob-h67sh","reconcileID":"0947c3eb-20af-480c-bd42-53733b6cb2dd","job":"test-ns-z95z9/test-preemption-trainjob-h67sh","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:30.489497753Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-rf2xr","reconcileID":"9b86bc7a-3e60-4136-9f0f-d2ef28a788a0"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:30.489521477Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-z95z9","name":"lq-6dt7b","reconcileID":"b4a83985-563a-4985-8d40-7f3a19ed8cae"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:30.539874475Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-z95z9","name":"test-preemption-trainjob-h67sh","reconcileID":"a84d46aa-75b2-4db7-b9be-68bbb5dd9454","job":"test-ns-z95z9/test-preemption-trainjob-h67sh","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:30.541901884Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-6dt7b","namespace":"test-ns-z95z9"}}
{"level":"Level(-2)","ts":"2026-11-02T19:26:30.541996284Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-z95z9","name":"lq-6dt7b","reconcileID":"54dcaeca-9e11-482c-a84a-f70ea81ea96b"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:30.5449336Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-rf2xr"}}
{"level":"Level(-2)","ts":"2026-11-02T19:26:30.545077848Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-rf2xr","reconcileID":"5cab25ed-dfcd-46ee-8285-815f4eb9d520"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:30.694591573Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-z95z9","name":"test-preemption-trainjob-h67sh","reconcileID":"bf278ba2-0f03-4452-bcee-fa5a71989f62","job":"test-ns-z95z9/test-preemption-trainjob-h67sh","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:30.849008822Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-z95z9","name":"test-preemption-trainjob-h67sh","reconcileID":"9f3c2741-4b52-4a00-bc85-98a5c9f27751","job":"test-ns-z95z9/test-preemption-trainjob-h67sh","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:31.004016227Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-z95z9","name":"test-preemption-trainjob-h67sh","reconcileID":"33dac72a-e78d-4cca-a655-3a63ec85dc8e","job":"test-ns-z95z9/test-preemption-trainjob-h67sh","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:31.742732911Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-z95z9","name":"trainjob-test-preemption-trainjob-h67sh-61b2a","reconcileID":"1ab891ae-eb66-4d5f-961a-fcdd59f7af9a"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:32.426858668Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-z95z9","name":"trainjob-test-preemption-trainjob-h67sh-61b2a","reconcileID":"dcb2866e-caec-4e22-a405-1651f46b9821"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:32.426883778Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-h67sh-61b2a","namespace":"test-ns-z95z9"},"queue":"lq-6dt7b","status":"admitted","clusterQueue":"cq-rf2xr"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:32.426919079Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-z95z9","name":"trainjob-test-preemption-trainjob-h67sh-61b2a","reconcileID":"dcb2866e-caec-4e22-a405-1651f46b9821","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T19:26:32.426939445Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1381","msg":"Workload will not be queued because the workload is not active","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-h67sh-61b2a","namespace":"test-ns-z95z9"},"queue":"lq-6dt7b","status":"admitted","clusterQueue":"cq-rf2xr"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:32.427012178Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-z95z9","name":"test-preemption-trainjob-h67sh","reconcileID":"e7fdc6fb-fe61-4cfe-a0f2-a2ed2fa65690","job":"test-ns-z95z9/test-preemption-trainjob-h67sh","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:32.427038155Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-z95z9","name":"trainjob-test-preemption-trainjob-h67sh-61b2a","reconcileID":"3a223c4b-c349-427d-b94c-c88cddcd3154"}
{"level":"debug","ts":"2026-11-02T19:26:32.536834254Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-test-preemption-trainjob-h67sh-61b2a","namespace":"test-ns-z95z9"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"EvictedDueToDeactivated","action":"EvictedDueToDeactivated","note":"The workload is deactivated"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:32.536848985Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-h67sh-61b2a","namespace":"test-ns-z95z9"},"queue":"lq-6dt7b","status":"admitted","clusterQueue":"cq-rf2xr"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:32.536869856Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-z95z9","name":"trainjob-test-preemption-trainjob-h67sh-61b2a","reconcileID":"b8de932f-5261-4bd5-bce8-fcd971b1594b"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:32.536906198Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-z95z9","name":"trainjob-test-preemption-trainjob-h67sh-61b2a","reconcileID":"b8de932f-5261-4bd5-bce8-fcd971b1594b","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T19:26:32.536905123Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1381","msg":"Workload will not be queued because the workload is not active","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-h67sh-61b2a","namespace":"test-ns-z95z9"},"queue":"lq-6dt7b","status":"admitted","clusterQueue":"cq-rf2xr"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:32.537002243Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-z95z9","name":"trainjob-test-preemption-trainjob-h67sh-61b2a","reconcileID":"b57120dc-43d7-40e3-b2eb-802c04a0ed84"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:32.582642786Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-z95z9","name":"test-preemption-trainjob-h67sh","reconcileID":"2e88bc13-37cf-4979-aeb0-947e6ea607c4","job":"test-ns-z95z9/test-preemption-trainjob-h67sh","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"info","ts":"2026-11-02T19:26:32.846668809Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-preemption-trainjob-h67sh","namespace":"test-ns-z95z9"},"namespace":"test-ns-z95z9","name":"test-preemption-trainjob-h67sh","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"a5098f94-590f-47a6-b732-24fed09fa6f8","replica-role":"leader"}
{"level":"error","ts":"2026-11-02T19:26:32.899475202Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-z95z9","name":"test-preemption-trainjob-h67sh","reconcileID":"2e88bc13-37cf-4979-aeb0-947e6ea607c4","error":"jobs are still active","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"debug","ts":"2026-11-02T19:26:32.899492311Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"test-preemption-trainjob-h67sh","namespace":"test-ns-z95z9"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"Stopped","action":"Stopped","note":"The workload is deactivated"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:32.899620156Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-z95z9","name":"test-preemption-trainjob-h67sh","reconcileID":"c11641bc-1754-4414-9526-1986d540914c","job":"test-ns-z95z9/test-preemption-trainjob-h67sh","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-11-02T19:26:33.054828819Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-z95z9","name":"test-preemption-trainjob-h67sh","reconcileID":"c11641bc-1754-4414-9526-1986d540914c","error":"jobs are still active","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:33.054954732Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-z95z9","name":"test-preemption-trainjob-h67sh","reconcileID":"1beb9966-5b9a-4bca-8576-f484ac3eadfa","job":"test-ns-z95z9/test-preemption-trainjob-h67sh","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:33.136799183Z","logger":"ElasticJobUngater","caller":"elasticjobs/elastic_job_ungater.go:112","msg":"Reconcile ElasticJobUngater","replica-role":"leader","namespace":"test-ns-z95z9","name":"trainjob-test-preemption-trainjob-h67sh-61b2a","reconcileID":"3925e6c7-03e1-46c1-b1d6-718a86531a1b"}
{"level":"error","ts":"2026-11-02T19:26:33.209969279Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-z95z9","name":"test-preemption-trainjob-h67sh","reconcileID":"1beb9966-5b9a-4bca-8576-f484ac3eadfa","error":"jobs are still active","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:33.210088607Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-z95z9","name":"test-preemption-trainjob-h67sh","reconcileID":"78a99c97-0d5e-4e7b-9a00-2631ddbe20b0","job":"test-ns-z95z9/test-preemption-trainjob-h67sh","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-11-02T19:26:33.36568022Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-z95z9","name":"test-preemption-trainjob-h67sh","reconcileID":"78a99c97-0d5e-4e7b-9a00-2631ddbe20b0","error":"jobs are still active","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:33.365793587Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-z95z9","name":"test-preemption-trainjob-h67sh","reconcileID":"0deba943-5aaf-4d9f-8ee7-7782b9560f95","job":"test-ns-z95z9/test-preemption-trainjob-h67sh","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:33.427341643Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-z95z9","name":"lq-6dt7b","reconcileID":"0dab96c7-56d2-465a-b803-ae29d0b9403c"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:33.427345263Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-rf2xr","reconcileID":"646c28de-6ad2-4e27-83c9-bc7f4d9f9678"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:33.479902299Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-6dt7b","namespace":"test-ns-z95z9"}}
{"level":"Level(-2)","ts":"2026-11-02T19:26:33.480035758Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-z95z9","name":"lq-6dt7b","reconcileID":"4acd5fdb-1a53-4560-8bc4-36e97d9aeb4d"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:33.48292722Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-rf2xr"}}
{"level":"Level(-2)","ts":"2026-11-02T19:26:33.483071011Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-rf2xr","reconcileID":"b14a86df-e00c-47f8-a4a0-e56e2ec5a5b3"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:33.490503069Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-rf2xr"}}
{"level":"Level(-2)","ts":"2026-11-02T19:26:33.490580862Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-rf2xr","reconcileID":"7c4cf921-a98a-4dbe-b069-228c709c45a4"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:33.490635652Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-z95z9","name":"trainjob-test-preemption-trainjob-h67sh-61b2a","reconcileID":"7111e06a-9bfe-459b-8116-bd4eb66a02aa"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:33.506670093Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:171","msg":"ResourceFlavor update event","replica-role":"leader","resourceFlavor":{"name":"rf-nkprn"}}
{"level":"Level(-2)","ts":"2026-11-02T19:26:33.506756198Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-nkprn","reconcileID":"9f97597d-f82f-45d1-9d32-c4b3609f6709"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:33.597056419Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:376","msg":"ClusterQueue delete event","replica-role":"leader","clusterQueue":{"name":"cq-rf2xr"}}
{"level":"Level(-2)","ts":"2026-11-02T19:26:33.597153138Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:385","msg":"Cleared resource metrics for deleted ClusterQueue.","replica-role":"leader","clusterQueue":{"name":"cq-rf2xr"}}
{"level":"Level(-2)","ts":"2026-11-02T19:26:33.597178058Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-z95z9","name":"trainjob-test-preemption-trainjob-h67sh-61b2a","reconcileID":"c7b421a1-37b4-4ada-bb25-7d73c2bdf5df"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:33.597177212Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-z95z9","name":"lq-6dt7b","reconcileID":"4449cf3c-bbba-4b08-bbcf-358adfece9da"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:33.597250161Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-nkprn","reconcileID":"a048bff0-a0e8-4b50-8112-264c9df6ff99"}
{"level":"info","ts":"2026-11-02T19:26:33.630537752Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-preemption-trainjob-h67sh","namespace":"test-ns-z95z9"},"namespace":"test-ns-z95z9","name":"test-preemption-trainjob-h67sh","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"a0b69103-e33d-4b84-a7cf-8252aa6aad2f","replica-role":"leader"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:33.650387171Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-6dt7b","namespace":"test-ns-z95z9"}}
{"level":"Level(-2)","ts":"2026-11-02T19:26:33.650466252Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-z95z9","name":"lq-6dt7b","reconcileID":"17e4727e-016c-4be6-871e-31f01293b81b"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:33.701187012Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:159","msg":"ResourceFlavor delete event","replica-role":"leader","resourceFlavor":{"name":"rf-nkprn"}}
{"level":"Level(-2)","ts":"2026-11-02T19:26:33.793149425Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-z95z9","name":"test-preemption-trainjob-h67sh","reconcileID":"d535a5b1-5292-40cc-8542-c9067b614a3e","job":"test-ns-z95z9/test-preemption-trainjob-h67sh","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:33.793357527Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-z95z9","name":"trainjob-test-preemption-trainjob-h67sh-61b2a","reconcileID":"12b77845-b280-484b-b555-8c29f7357286"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:33.793392559Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-z95z9","name":"trainjob-test-preemption-trainjob-h67sh-61b2a","reconcileID":"12b77845-b280-484b-b555-8c29f7357286","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T19:26:33.793386679Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-h67sh-61b2a","namespace":"test-ns-z95z9"},"queue":"lq-6dt7b","status":"pending","prevStatus":"admitted","prevClusterQueue":"cq-rf2xr"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:33.793444729Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1381","msg":"Workload will not be queued because the workload is not active","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-h67sh-61b2a","namespace":"test-ns-z95z9"},"queue":"lq-6dt7b","status":"pending","prevStatus":"admitted","prevClusterQueue":"cq-rf2xr"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:33.793527241Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-z95z9","name":"trainjob-test-preemption-trainjob-h67sh-61b2a","reconcileID":"bf4e0d4f-6129-46a5-9398-81db111dc413"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:33.802390163Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:141","msg":"ResourceFlavor create event","replica-role":"leader","resourceFlavor":{"name":"rf-rzn84"}}
{"level":"Level(-2)","ts":"2026-11-02T19:26:33.802469312Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-rzn84","reconcileID":"ed02bfbe-5365-4036-bea8-dedfebd7a460"}
{"level":"error","ts":"2026-11-02T19:26:33.841428994Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-z95z9","name":"test-preemption-trainjob-h67sh","reconcileID":"d535a5b1-5292-40cc-8542-c9067b614a3e","job":"test-ns-z95z9/test-preemption-trainjob-h67sh","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-h67sh\" is forbidden: unable to create new content in namespace test-ns-z95z9 because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-02T19:26:33.84149522Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-z95z9","name":"test-preemption-trainjob-h67sh","reconcileID":"d535a5b1-5292-40cc-8542-c9067b614a3e","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-h67sh\" is forbidden: unable to create new content in namespace test-ns-z95z9 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:33.841597387Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-z95z9","name":"test-preemption-trainjob-h67sh","reconcileID":"11e1cf41-88ea-47b5-b545-28d34587fd06","job":"test-ns-z95z9/test-preemption-trainjob-h67sh","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-11-02T19:26:33.890462483Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-z95z9","name":"test-preemption-trainjob-h67sh","reconcileID":"11e1cf41-88ea-47b5-b545-28d34587fd06","job":"test-ns-z95z9/test-preemption-trainjob-h67sh","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-h67sh\" is forbidden: unable to create new content in namespace test-ns-z95z9 because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-02T19:26:33.890532743Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-z95z9","name":"test-preemption-trainjob-h67sh","reconcileID":"11e1cf41-88ea-47b5-b545-28d34587fd06","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-h67sh\" is forbidden: unable to create new content in namespace test-ns-z95z9 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:33.90087729Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-z95z9","name":"test-preemption-trainjob-h67sh","reconcileID":"56d39df9-b679-40e3-a4fb-c3479fa0a079","job":"test-ns-z95z9/test-preemption-trainjob-h67sh","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:33.905047469Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-h67sh-61b2a","namespace":"test-ns-z95z9"},"queue":"lq-6dt7b","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:33.905073296Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-z95z9","name":"trainjob-test-preemption-trainjob-h67sh-61b2a","reconcileID":"55a3a7e1-d0ec-4ff9-8284-bceb7d7e3b31"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:33.90510434Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1381","msg":"Workload will not be queued because the workload is not active","replica-role":"leader","workload":{"name":"trainjob-test-preemption-trainjob-h67sh-61b2a","namespace":"test-ns-z95z9"},"queue":"lq-6dt7b","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:33.905101771Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-z95z9","name":"trainjob-test-preemption-trainjob-h67sh-61b2a","reconcileID":"55a3a7e1-d0ec-4ff9-8284-bceb7d7e3b31","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T19:26:33.905203617Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-z95z9","name":"trainjob-test-preemption-trainjob-h67sh-61b2a","reconcileID":"fef60e4f-6b69-445e-82d3-1db9d6ac1337"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:33.924171714Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:351","msg":"ClusterQueue create event","replica-role":"leader","clusterQueue":{"name":"cq-ccnzs"}}
{"level":"Level(-2)","ts":"2026-11-02T19:26:33.924363423Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-ccnzs","reconcileID":"546d4756-c32c-4f47-b7b3-bcdb9b053ebc"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:33.942179008Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:240","msg":"LocalQueue create event","replica-role":"leader","localQueue":{"name":"lq-psq8h","namespace":"test-ns-292fh"}}
{"level":"Level(-2)","ts":"2026-11-02T19:26:33.942333398Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-292fh","name":"lq-psq8h","reconcileID":"12355572-c3e2-4670-a961-a097f809c9ec"}
{"level":"error","ts":"2026-11-02T19:26:33.949322345Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-z95z9","name":"test-preemption-trainjob-h67sh","reconcileID":"56d39df9-b679-40e3-a4fb-c3479fa0a079","job":"test-ns-z95z9/test-preemption-trainjob-h67sh","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-h67sh\" is forbidden: unable to create new content in namespace test-ns-z95z9 because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-02T19:26:33.949386181Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-z95z9","name":"test-preemption-trainjob-h67sh","reconcileID":"56d39df9-b679-40e3-a4fb-c3479fa0a079","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-h67sh\" is forbidden: unable to create new content in namespace test-ns-z95z9 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:33.949487718Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-z95z9","name":"test-preemption-trainjob-h67sh","reconcileID":"e6705281-d339-487b-a772-e01b6dea15b4","job":"test-ns-z95z9/test-preemption-trainjob-h67sh","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:33.978555805Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-ccnzs"}}
{"level":"Level(-2)","ts":"2026-11-02T19:26:33.97874004Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-ccnzs","reconcileID":"dfde451a-308c-4621-bd0d-874f7beb7714"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:33.994095915Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-292fh","name":"lq-psq8h","reconcileID":"8f38dc6e-f396-49b3-89aa-4bb79ac77357"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:33.994089835Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-psq8h","namespace":"test-ns-292fh"}}
{"level":"error","ts":"2026-11-02T19:26:33.99762549Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-z95z9","name":"test-preemption-trainjob-h67sh","reconcileID":"e6705281-d339-487b-a772-e01b6dea15b4","job":"test-ns-z95z9/test-preemption-trainjob-h67sh","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-h67sh\" is forbidden: unable to create new content in namespace test-ns-z95z9 because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-02T19:26:33.997689182Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-z95z9","name":"test-preemption-trainjob-h67sh","reconcileID":"e6705281-d339-487b-a772-e01b6dea15b4","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-h67sh\" is forbidden: unable to create new content in namespace test-ns-z95z9 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:34.038330586Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-z95z9","name":"test-preemption-trainjob-h67sh","reconcileID":"5d4fcbdf-6e9b-472b-8c85-59f3eb721924","job":"test-ns-z95z9/test-preemption-trainjob-h67sh","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:34.048424324Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-292fh","name":"lq-psq8h","reconcileID":"1bcc7f63-906a-4f96-8d86-07f34e66456f"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:34.048478237Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-psq8h","namespace":"test-ns-292fh"}}
{"level":"info","ts":"2026-11-02T19:26:34.06729137Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:117","msg":"Validating create","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-runtimepatches-trainjob-99mcp","namespace":"test-ns-292fh"},"namespace":"test-ns-292fh","name":"test-runtimepatches-trainjob-99mcp","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:admin","requestID":"1107ad62-18cf-42d6-8f5b-4626645a7195","replica-role":"leader"}
{"level":"error","ts":"2026-11-02T19:26:34.086802971Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-z95z9","name":"test-preemption-trainjob-h67sh","reconcileID":"5d4fcbdf-6e9b-472b-8c85-59f3eb721924","job":"test-ns-z95z9/test-preemption-trainjob-h67sh","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-h67sh\" is forbidden: unable to create new content in namespace test-ns-z95z9 because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-02T19:26:34.086875811Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-z95z9","name":"test-preemption-trainjob-h67sh","reconcileID":"5d4fcbdf-6e9b-472b-8c85-59f3eb721924","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-h67sh\" is forbidden: unable to create new content in namespace test-ns-z95z9 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-3)","ts":"2026-11-02T19:26:34.098334689Z","logger":"localqueue-reconciler","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-292fh","name":"lq-psq8h","reconcileID":"1bcc7f63-906a-4f96-8d86-07f34e66456f","error":"Operation cannot be fulfilled on localqueues.kueue.x-k8s.io \"lq-psq8h\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:34.098439732Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-292fh","name":"lq-psq8h","reconcileID":"9bf8d0c0-677f-40bf-8f83-405e9a2b694a"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:34.117230187Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-292fh","name":"test-runtimepatches-trainjob-99mcp","reconcileID":"64ffbb22-0c91-4d8a-81e3-ce9bd7fcf405","job":"test-ns-292fh/test-runtimepatches-trainjob-99mcp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:34.430666474Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1276","msg":"Workload create event","replica-role":"leader","workload":{"name":"trainjob-test-runtimepatches-trainjob-99mcp-1af38","namespace":"test-ns-292fh"},"queue":"lq-psq8h","status":"pending"}
{"level":"debug","ts":"2026-11-02T19:26:34.430698791Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"test-runtimepatches-trainjob-99mcp","namespace":"test-ns-292fh"},"kind":"TrainJob","apiVersion":"trainer.kubeflow.org/v1alpha1","type":"Normal","reason":"CreatedWorkload","action":"CreatedWorkload","note":"Created Workload: test-ns-292fh/trainjob-test-runtimepatches-trainjob-99mcp-1af38"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:34.430716074Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-z95z9","name":"test-preemption-trainjob-h67sh","reconcileID":"167ad2e8-f095-4579-b51a-19352f5ab3f9","job":"test-ns-z95z9/test-preemption-trainjob-h67sh","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:34.430781593Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-292fh","name":"trainjob-test-runtimepatches-trainjob-99mcp-1af38","reconcileID":"f6932939-c4a6-4daf-8828-27bb75656f1d"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:34.430820693Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-292fh","name":"trainjob-test-runtimepatches-trainjob-99mcp-1af38","reconcileID":"f6932939-c4a6-4daf-8828-27bb75656f1d","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T19:26:34.430870297Z","logger":"scheduler","caller":"scheduler/scheduler.go:308","msg":"Obtained heads","replica-role":"leader","schedulingCycle":4,"headCount":1,"waitDuration":4.941729555}
{"level":"Level(-2)","ts":"2026-11-02T19:26:34.430934306Z","logger":"scheduler","caller":"scheduler/scheduler.go:323","msg":"Snapshot taken","replica-role":"leader","schedulingCycle":4,"duration":0.000015775}
{"level":"Level(-2)","ts":"2026-11-02T19:26:34.430894213Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-292fh","name":"trainjob-test-runtimepatches-trainjob-99mcp-1af38","reconcileID":"18705339-0742-415b-8430-81cac7be77b3"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:34.431035151Z","logger":"scheduler","caller":"scheduler/scheduler.go:328","msg":"Nomination done","replica-role":"leader","schedulingCycle":4,"entries":1,"inadmissibleEntries":0,"duration":0.000086147}
{"level":"Level(-2)","ts":"2026-11-02T19:26:34.431057411Z","logger":"scheduler","caller":"scheduler/scheduler.go:406","msg":"Attempting to schedule workload","replica-role":"leader","schedulingCycle":4,"workload":{"name":"trainjob-test-runtimepatches-trainjob-99mcp-1af38","namespace":"test-ns-292fh"},"clusterQueue":{"name":"cq-ccnzs"}}
{"level":"Level(-2)","ts":"2026-11-02T19:26:34.43116126Z","logger":"scheduler","caller":"scheduler/scheduler.go:1041","msg":"Workload assumed in the cache","replica-role":"leader","schedulingCycle":4,"workload":{"name":"trainjob-test-runtimepatches-trainjob-99mcp-1af38","namespace":"test-ns-292fh"},"clusterQueue":{"name":"cq-ccnzs"}}
{"level":"Level(-2)","ts":"2026-11-02T19:26:34.431187044Z","logger":"scheduler","caller":"scheduler/scheduler.go:362","msg":"Workload processing done","replica-role":"leader","schedulingCycle":4,"duration":0.000135343}
{"level":"Level(-2)","ts":"2026-11-02T19:26:34.431214234Z","logger":"scheduler","caller":"scheduler/scheduler.go:297","msg":"Scheduling cycle complete","replica-role":"leader","schedulingCycle":4,"duration":4.942073582}
{"level":"Level(-2)","ts":"2026-11-02T19:26:34.431239645Z","logger":"scheduler","caller":"scheduler/scheduler.go:295","msg":"Scheduling cycle starts","replica-role":"leader","schedulingCycle":5}
{"level":"error","ts":"2026-11-02T19:26:34.478884434Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-z95z9","name":"test-preemption-trainjob-h67sh","reconcileID":"167ad2e8-f095-4579-b51a-19352f5ab3f9","job":"test-ns-z95z9/test-preemption-trainjob-h67sh","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-h67sh\" is forbidden: unable to create new content in namespace test-ns-z95z9 because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-02T19:26:34.478956661Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-z95z9","name":"test-preemption-trainjob-h67sh","reconcileID":"167ad2e8-f095-4579-b51a-19352f5ab3f9","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-h67sh\" is forbidden: unable to create new content in namespace test-ns-z95z9 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:34.479071473Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-292fh","name":"test-runtimepatches-trainjob-99mcp","reconcileID":"8f84c839-2475-4b6e-9025-2b0c55634993","job":"test-ns-292fh/test-runtimepatches-trainjob-99mcp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:34.538776938Z","logger":"scheduler","caller":"scheduler/scheduler.go:1000","msg":"Workload successfully admitted and assigned flavors","replica-role":"leader","schedulingCycle":4,"workload":{"name":"trainjob-test-runtimepatches-trainjob-99mcp-1af38","namespace":"test-ns-292fh"},"clusterQueue":{"name":"cq-ccnzs"},"assignments":[{"name":"node","flavors":{"cpu":"rf-rzn84","memory":"rf-rzn84"},"resourceUsage":{"cpu":"100m","memory":"128Mi"},"count":1}]}
{"level":"Level(-2)","ts":"2026-11-02T19:26:34.538813234Z","logger":"multikueue-workload","caller":"multikueue/workload.go:184","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-292fh","name":"trainjob-test-runtimepatches-trainjob-99mcp-1af38","reconcileID":"4847e2a9-8201-4cd6-bd3f-5609d60129f5"}
{"level":"debug","ts":"2026-11-02T19:26:34.538815273Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-test-runtimepatches-trainjob-99mcp-1af38","namespace":"test-ns-292fh"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"QuotaReserved","action":"QuotaReserved","note":"Quota reserved in ClusterQueue cq-ccnzs, wait time since queued was 1s"}
{"level":"debug","ts":"2026-11-02T19:26:34.538852104Z","logger":"events","caller":"events/event_broadcaster.go:338","msg":"Event occurred","object":{"name":"trainjob-test-runtimepatches-trainjob-99mcp-1af38","namespace":"test-ns-292fh"},"kind":"Workload","apiVersion":"kueue.x-k8s.io/v1beta2","type":"Normal","reason":"Admitted","action":"Admitted","note":"Admitted by ClusterQueue cq-ccnzs, wait time since reservation was 1s"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:34.538856381Z","logger":"multikueue-workload","caller":"multikueue/workload.go:210","msg":"Skip Workload","replica-role":"leader","namespace":"test-ns-292fh","name":"trainjob-test-runtimepatches-trainjob-99mcp-1af38","reconcileID":"4847e2a9-8201-4cd6-bd3f-5609d60129f5","isDeleted":false}
{"level":"Level(-2)","ts":"2026-11-02T19:26:34.538876777Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"leader","workload":{"name":"trainjob-test-runtimepatches-trainjob-99mcp-1af38","namespace":"test-ns-292fh"},"queue":"lq-psq8h","status":"admitted","prevStatus":"pending","clusterQueue":"cq-ccnzs"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:34.53907008Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-292fh","name":"trainjob-test-runtimepatches-trainjob-99mcp-1af38","reconcileID":"bee8e02e-8074-42e2-9384-6cbc927770cf"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:34.63581395Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-292fh","name":"test-runtimepatches-trainjob-99mcp","reconcileID":"e9bf6bb4-16ce-4486-8cdf-b2e853d28b1e","job":"test-ns-292fh/test-runtimepatches-trainjob-99mcp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:34.790659136Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-292fh","name":"test-runtimepatches-trainjob-99mcp","reconcileID":"e9bf6bb4-16ce-4486-8cdf-b2e853d28b1e","job":"test-ns-292fh/test-runtimepatches-trainjob-99mcp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:34.794661676Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-z95z9","name":"lq-6dt7b","reconcileID":"d4452411-8444-43e3-9b07-109630e69ae2"}
{"level":"info","ts":"2026-11-02T19:26:34.901234599Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-runtimepatches-trainjob-99mcp","namespace":"test-ns-292fh"},"namespace":"test-ns-292fh","name":"test-runtimepatches-trainjob-99mcp","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"f271f362-c118-4746-9144-686c7a357fec","replica-role":"leader"}
{"level":"Level(-3)","ts":"2026-11-02T19:26:35.009510033Z","logger":"trainjob","caller":"jobframework/reconciler.go:594","msg":"Unsuspending job","replica-role":"leader","namespace":"test-ns-292fh","name":"test-runtimepatches-trainjob-99mcp","reconcileID":"e9bf6bb4-16ce-4486-8cdf-b2e853d28b1e","job":"test-ns-292fh/test-runtimepatches-trainjob-99mcp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-runtimepatches-trainjob-99mcp\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-3)","ts":"2026-11-02T19:26:35.009604332Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-292fh","name":"test-runtimepatches-trainjob-99mcp","reconcileID":"e9bf6bb4-16ce-4486-8cdf-b2e853d28b1e","error":"Operation cannot be fulfilled on trainjobs.trainer.kubeflow.org \"test-runtimepatches-trainjob-99mcp\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:35.009709585Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-z95z9","name":"test-preemption-trainjob-h67sh","reconcileID":"3797d28a-7e6b-4cdf-9c60-b008d30ff2c2","job":"test-ns-z95z9/test-preemption-trainjob-h67sh","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"error","ts":"2026-11-02T19:26:35.058066387Z","logger":"trainjob","caller":"jobframework/reconciler.go:1118","msg":"Unable to list child workloads","replica-role":"leader","namespace":"test-ns-z95z9","name":"test-preemption-trainjob-h67sh","reconcileID":"3797d28a-7e6b-4cdf-9c60-b008d30ff2c2","job":"test-ns-z95z9/test-preemption-trainjob-h67sh","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-h67sh\" is forbidden: unable to create new content in namespace test-ns-z95z9 because it is being terminated","stacktrace":"sigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ensureOneWorkload\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:1118\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*JobReconciler).ReconcileGenericJob\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:401\nsigs.k8s.io/kueue/pkg/controller/jobframework.(*genericReconciler).Reconcile\n\t/workspace/upstream/kueue/src/pkg/controller/jobframework/reconciler.go:2024\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-02T19:26:35.058163058Z","logger":"trainjob","caller":"controller/controller.go:494","msg":"Reconciler error","replica-role":"leader","namespace":"test-ns-z95z9","name":"test-preemption-trainjob-h67sh","reconcileID":"3797d28a-7e6b-4cdf-9c60-b008d30ff2c2","error":"workloads.kueue.x-k8s.io \"test-preemption-trainjob-h67sh\" is forbidden: unable to create new content in namespace test-ns-z95z9 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/workspace/upstream/kueue/src/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:312"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:35.058272345Z","logger":"trainjob","caller":"jobframework/reconciler.go:398","msg":"Reconciling Job","replica-role":"leader","namespace":"test-ns-292fh","name":"test-runtimepatches-trainjob-99mcp","reconcileID":"12f7bb5d-e798-4a03-a536-bd42c228ee7b","job":"test-ns-292fh/test-runtimepatches-trainjob-99mcp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:35.195254416Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-ccnzs"}}
{"level":"Level(-2)","ts":"2026-11-02T19:26:35.195355371Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-ccnzs","reconcileID":"42cb5bdf-a9d3-4548-9c1d-739b356f9b46"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:35.195448958Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-292fh","name":"trainjob-test-runtimepatches-trainjob-99mcp-1af38","reconcileID":"dc0c0161-36e1-47d3-8df0-8bd545974b6e"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:35.211186173Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:171","msg":"ResourceFlavor update event","replica-role":"leader","resourceFlavor":{"name":"rf-rzn84"}}
{"level":"Level(-2)","ts":"2026-11-02T19:26:35.211283299Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:95","msg":"Reconcile ResourceFlavor","replica-role":"leader","name":"rf-rzn84","reconcileID":"0b6c690f-0cce-41b1-a600-40ed3a77a4c5"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:35.213075688Z","logger":"trainjob","caller":"jobframework/reconciler.go:591","msg":"Job admitted, unsuspending","replica-role":"leader","namespace":"test-ns-292fh","name":"test-runtimepatches-trainjob-99mcp","reconcileID":"12f7bb5d-e798-4a03-a536-bd42c228ee7b","job":"test-ns-292fh/test-runtimepatches-trainjob-99mcp","gvk":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:35.250343612Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"leader","clusterQueue":{"name":"cq-ccnzs"}}
{"level":"Level(-2)","ts":"2026-11-02T19:26:35.25041762Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-292fh","name":"lq-psq8h","reconcileID":"22e3867e-9fa1-4658-962c-e82f966b9208"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:35.250435552Z","logger":"clusterqueue-reconciler","caller":"core/clusterqueue_controller.go:168","msg":"Reconcile ClusterQueue","replica-role":"leader","name":"cq-ccnzs","reconcileID":"69f89852-8e15-4830-b2be-3da6383eabb0"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:35.250464375Z","logger":"workload-reconciler","caller":"core/workload_controller.go:277","msg":"Reconcile Workload","replica-role":"leader","namespace":"test-ns-292fh","name":"trainjob-test-runtimepatches-trainjob-99mcp-1af38","reconcileID":"c4408bb0-342e-4d04-a944-2f3fc960ba01"}
{"level":"Level(-2)","ts":"2026-11-02T19:26:35.302739171Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"leader","localQueue":{"name":"lq-psq8h","namespace":"test-ns-292fh"}}
{"level":"Level(-2)","ts":"2026-11-02T19:26:35.302869614Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:186","msg":"Reconcile LocalQueue","replica-role":"leader","namespace":"test-ns-292fh","name":"lq-psq8h","reconcileID":"97ed98ba-0316-4a42-a99a-33e220bc157b"}
{"level":"info","ts":"2026-11-02T19:26:35.325532267Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-runtimepatches-trainjob-99mcp","namespace":"test-ns-292fh"},"namespace":"test-ns-292fh","name":"test-runtimepatches-trainjob-99mcp","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"0ea8ec7a-a4f7-41d7-be64-48e27690577f","replica-role":"leader"}