{"level":"info","ts":"2026-11-03T12:58:48.007752191Z","logger":"setup","caller":"kueue/main.go:721","msg":"Configuration loaded","config":"apiVersion: config.kueue.x-k8s.io/v1beta2\nclientConnection:\n  burst: 100\n  qps: 50\ncontroller:\n  groupKindConcurrency:\n    ClusterQueue.kueue.x-k8s.io: 1\n    Job.batch: 5\n    LocalQueue.kueue.x-k8s.io: 1\n    Pod: 5\n    ResourceFlavor.kueue.x-k8s.io: 1\n    Workload.kueue.x-k8s.io: 5\nhealth:\n  healthProbeBindAddress: :8081\nintegrations:\n  frameworks:\n  - trainer.kubeflow.org/trainjob\n  - deployment\ninternalCertManagement:\n  enable: false\nkind: Configuration\nleaderElection:\n  leaderElect: true\n  leaseDuration: 2m17s\n  renewDeadline: 1m47s\n  resourceLock: leases\n  resourceName: c1f6bfd2.kueue.x-k8s.io\n  resourceNamespace: \"\"\n  retryPeriod: 26s\nmanageJobsWithoutQueueName: false\nmanagedJobsNamespaceSelector:\n  matchLabels:\n    kueue.openshift.io/managed: \"true\"\nmetrics:\n  bindAddress: :8443\n  enableClusterQueueResources: true\n  localQueueMetrics:\n    enable: true\nmultiKueue:\n  dispatcherName: kueue.x-k8s.io/multikueue-dispatcher-all-at-once\n  gcInterval: 1m0s\n  origin: multikueue\n  workerLostTimeout: 15m0s\nnamespace: openshift-kueue-operator\ntls:\n  cipherSuites:\n  - TLS_AES_128_GCM_SHA256\n  - TLS_AES_256_GCM_SHA384\n  - TLS_CHACHA20_POLY1305_SHA256\n  - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256\n  - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256\n  - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384\n  - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\n  - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256\n  - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256\n  minVersion: VersionTLS12\nvisibilityServer:\n  bindPort: 8082\nwebhook:\n  certDir: /tmp/k8s-webhook-server/serving-certs\n  port: 9443\n"}
{"level":"info","ts":"2026-11-03T12:58:48.007952924Z","logger":"setup","caller":"kueue/main.go:170","msg":"Initializing","gitVersion":"d12900d9-dirty","gitCommit":"d12900d94cd6813a3882a6391ffd70009a4b61d9","buildDate":"2026-10-05T16:41:45Z"}
{"level":"Level(-2)","ts":"2026-11-03T12:58:48.008354056Z","logger":"setup","caller":"features/kube_features.go:1144","msg":"Loaded feature gates","featureGates":{"AdmissionFairSharing":true,"AdmissionGatedBy":false,"AssignQueueLabelsForPods":true,"CleanupProvisioningRequestsOnEviction":true,"ConcurrentAdmission":false,"CustomMetricLabels":false,"DRAExtendedResources":false,"DeferRayServiceFinalizationForRedisCleanup":true,"DeploymentJobUIDLabel":false,"DynamicResourceAllocation":false,"ElasticJobsViaWorkloadSlices":true,"ElasticJobsViaWorkloadSlicesWithTAS":false,"EnforceProvisioningPodTemplateContents":false,"FailureRecoveryPolicy":false,"FairSharingPreemptWithinNominal":true,"FairSharingPrioritizeNonBorrowing":true,"FairSharingReevaluatePreemptionCandidates":false,"FastQuotaReleaseInPodIntegration":false,"FinalizeTerminatingPodGroups":false,"FinishOrphanedWorkloads":true,"FlavorFungibility":true,"FlavorFungibilityPreserveScanProgress":true,"HierarchicalCohorts":true,"HighMaxParallelismWithinReconcile":false,"KueueDRAIntegration":true,"KueueDRAIntegrationExtendedResource":false,"KueueDRAIntegrationPartitionableDevices":false,"KueueDRARejectWorkloadsWhenDRADisabled":true,"LWSImmutableGroupSize":true,"LendingLimit":true,"LocalQueueDefaulting":true,"LocalQueueMetrics":true,"ManagedJobsNamespaceSelectorAlwaysRespected":true,"MetricForWorkloadCreationLatency":true,"MetricsForCohorts":true,"MultiKueue":true,"MultiKueueAdaptersForCustomJobs":true,"MultiKueueAllowInsecureKubeconfigs":false,"MultiKueueBatchJobClearingTTLSecondsAfterFinishedOnWorkerCluster":false,"MultiKueueBatchJobWithManagedBy":true,"MultiKueueClusterProfile":false,"MultiKueueManagerQuotaAutomation":false,"MultiKueueOrchestratedPreemption":false,"MultiKueueRedoAdmissionOnEvictionInWorker":true,"MultiKueueRemoteSpecSync":true,"MultiKueueReuseClientConnectionConfigForWorkers":false,"MultiKueueWaitForWorkloadAdmitted":true,"ObjectRetentionPolicies":true,"PartialAdmission":true,"PodGroupSchedulingShapeOrdering":false,"PodIntegrationCountSucceededPodsAsReady":false,"PodIntegrationValidateGroupOwner":true,"PrioritizePreemptorWorkloads":false,"PriorityBoost":false,"PrioritySortingWithinCohort":true,"PropagateBatchJobLabelsToWorkload":true,"QuotaCheckStrategy":false,"ReclaimablePods":true,"RecomputeAssignmentUponPreemptionTargetsOverlap":false,"RejectUpdatesToCQWithInvalidOnFlavors":false,"RemoveFinalizersWithStrictPatch":true,"ReservedResourceNameValidation":false,"SanitizePodSets":true,"SchedulerLongRequeueInterval":false,"SchedulerTimestampPreemptionBuffer":false,"SchedulingEquivalenceHashing":true,"SchedulingEquivalenceHashingIgnorePodSetName":false,"ShortWorkloadNames":false,"SkipAncestorCheckForDeletedWorkloads":true,"SkipChildJobSuspension":false,"SkipFinalizersForPodsSuspendedByParent":true,"SkipReassignmentForPodOwnedWorkloads":false,"SparkApplicationIntegration":false,"TASBalancedPlacement":false,"TASCacheNodeMatchResults":true,"TASCacheTopologyTree":false,"TASCachingRemainingResources":true,"TASFailedNodeReplacement":true,"TASFailedNodeReplacementFailFast":true,"TASGroupedPodSetSlicing":false,"TASHandleOverlappingFlavors":true,"TASMultiLayerTopology":false,"TASPartialSlices":true,"TASProfileMixed":true,"TASRecomputeAssignmentWithinSchedulingCycle":true,"TASRejectFalseUnconstrainedTopology":false,"TASReplaceNodeDueToNotReadyOverFixedTime":true,"TASReplaceNodeOnNodeTaints":true,"TASReplaceNodeOnPodTermination":true,"TASRespectNodeAffinityPreferred":false,"TASValidateWorkloadSliceSize":true,"TLSOptions":true,"TopologyAwareScheduling":true,"UnadmittedWorkloadsExplicitStatus":false,"UnadmittedWorkloadsObservability":false,"ValidateRayAndSparkJobUpdates":true,"VectorizedResourceRequests":true,"VisibilityOnDemand":true,"WorkloadIdentifierAnnotations":true,"WorkloadPriorityClassDefaulting":false,"WorkloadRequestUseMergePatch":false,"WorkloadValidateResourcesAreNonNegative":true,"WorkloadValidationForPodSetMetadata":true}}
{"level":"info","ts":"2026-11-03T12:58:48.008433731Z","logger":"setup","caller":"kueue/main.go:213","msg":"Initializing metrics certificate watcher using provided certificates","metrics-cert-path":"/etc/kueue/metrics/certs"}
{"level":"info","ts":"2026-11-03T12:58:48.008727607Z","logger":"controller-runtime.certwatcher","caller":"certwatcher/certwatcher.go:214","msg":"Updated current TLS certificate","cert":"/etc/kueue/metrics/certs/tls.crt","key":"/etc/kueue/metrics/certs/tls.key"}
{"level":"Level(-2)","ts":"2026-11-03T12:58:48.009015053Z","logger":"setup","caller":"kueue/main.go:257","msg":"K8S Client","qps":50,"burst":100}
{"level":"info","ts":"2026-11-03T12:58:48.318885834Z","logger":"setup","caller":"kueue/main.go:683","msg":"RoleTracker: leader election enabled"}
{"level":"error","ts":"2026-11-03T12:58:48.36948976Z","logger":"setup","caller":"kueue/main.go:445","msg":"Skipping admission check controller setup: Provisioning Requests not supported (Possible cause: missing or unsupported cluster-autoscaler)","error":"no matches for kind \"ProvisioningRequest\" in version \"autoscaling.x-k8s.io/v1\"","stacktrace":"main.setupIndexes\n\t/workspace/upstream/kueue/src/cmd/kueue/main.go:445\nmain.main\n\t/workspace/upstream/kueue/src/cmd/kueue/main.go:363\nruntime.main\n\t/usr/lib/golang/src/runtime/proc.go:290"}
{"level":"info","ts":"2026-11-03T12:58:48.419281725Z","logger":"setup","caller":"kueue/main.go:625","msg":"Probe endpoints are configured on healthz and readyz"}
{"level":"info","ts":"2026-11-03T12:58:48.41938946Z","logger":"queue-manager","caller":"queue/manager.go:138","msg":"DRA reconcile channel connected"}
{"level":"info","ts":"2026-11-03T12:58:48.469945692Z","logger":"setup","caller":"kueue/main.go:497","msg":"Skipping provisioning controller setup: Provisioning Requests not supported (Possible cause: missing or unsupported cluster-autoscaler)"}
{"level":"info","ts":"2026-11-03T12:58:48.470178118Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"workload.codeflare.dev/v1beta2, Kind=AppWrapper","path":"/mutate-workload-codeflare-dev-v1beta2-appwrapper"}
{"level":"info","ts":"2026-11-03T12:58:48.470269219Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-workload-codeflare-dev-v1beta2-appwrapper"}
{"level":"info","ts":"2026-11-03T12:58:48.47029859Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"workload.codeflare.dev/v1beta2, Kind=AppWrapper","path":"/validate-workload-codeflare-dev-v1beta2-appwrapper"}
{"level":"info","ts":"2026-11-03T12:58:48.47033472Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-workload-codeflare-dev-v1beta2-appwrapper"}
{"level":"info","ts":"2026-11-03T12:58:48.470430787Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"apps/v1, Kind=Deployment","path":"/mutate-apps-v1-deployment"}
{"level":"info","ts":"2026-11-03T12:58:48.470475104Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-apps-v1-deployment"}
{"level":"info","ts":"2026-11-03T12:58:48.47049663Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"apps/v1, Kind=Deployment","path":"/validate-apps-v1-deployment"}
{"level":"info","ts":"2026-11-03T12:58:48.470530045Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-apps-v1-deployment"}
{"level":"info","ts":"2026-11-03T12:58:48.470588621Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"batch/v1, Kind=Job","path":"/mutate-batch-v1-job"}
{"level":"info","ts":"2026-11-03T12:58:48.470633325Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-batch-v1-job"}
{"level":"info","ts":"2026-11-03T12:58:48.470654215Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"batch/v1, Kind=Job","path":"/validate-batch-v1-job"}
{"level":"info","ts":"2026-11-03T12:58:48.470688682Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-batch-v1-job"}
{"level":"info","ts":"2026-11-03T12:58:48.470741936Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"jobset.x-k8s.io/v1alpha2, Kind=JobSet","path":"/mutate-jobset-x-k8s-io-v1alpha2-jobset"}
{"level":"info","ts":"2026-11-03T12:58:48.470777469Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-jobset-x-k8s-io-v1alpha2-jobset"}
{"level":"info","ts":"2026-11-03T12:58:48.470799537Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"jobset.x-k8s.io/v1alpha2, Kind=JobSet","path":"/validate-jobset-x-k8s-io-v1alpha2-jobset"}
{"level":"info","ts":"2026-11-03T12:58:48.470831806Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-jobset-x-k8s-io-v1alpha2-jobset"}
{"level":"info","ts":"2026-11-03T12:58:48.470878895Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"kubeflow.org/v1, Kind=JAXJob","path":"/mutate-kubeflow-org-v1-jaxjob"}
{"level":"info","ts":"2026-11-03T12:58:48.470914239Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-kubeflow-org-v1-jaxjob"}
{"level":"info","ts":"2026-11-03T12:58:48.470936592Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"kubeflow.org/v1, Kind=JAXJob","path":"/validate-kubeflow-org-v1-jaxjob"}
{"level":"info","ts":"2026-11-03T12:58:48.470971715Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-kubeflow-org-v1-jaxjob"}
{"level":"info","ts":"2026-11-03T12:58:48.47102013Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"kubeflow.org/v1, Kind=PaddleJob","path":"/mutate-kubeflow-org-v1-paddlejob"}
{"level":"info","ts":"2026-11-03T12:58:48.471053347Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-kubeflow-org-v1-paddlejob"}
{"level":"info","ts":"2026-11-03T12:58:48.471073972Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"kubeflow.org/v1, Kind=PaddleJob","path":"/validate-kubeflow-org-v1-paddlejob"}
{"level":"info","ts":"2026-11-03T12:58:48.471106181Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-kubeflow-org-v1-paddlejob"}
{"level":"info","ts":"2026-11-03T12:58:48.471170415Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"kubeflow.org/v1, Kind=PyTorchJob","path":"/mutate-kubeflow-org-v1-pytorchjob"}
{"level":"info","ts":"2026-11-03T12:58:48.47120489Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-kubeflow-org-v1-pytorchjob"}
{"level":"info","ts":"2026-11-03T12:58:48.471226544Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"kubeflow.org/v1, Kind=PyTorchJob","path":"/validate-kubeflow-org-v1-pytorchjob"}
{"level":"info","ts":"2026-11-03T12:58:48.471265861Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-kubeflow-org-v1-pytorchjob"}
{"level":"info","ts":"2026-11-03T12:58:48.471314801Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"kubeflow.org/v1, Kind=TFJob","path":"/mutate-kubeflow-org-v1-tfjob"}
{"level":"info","ts":"2026-11-03T12:58:48.471354911Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-kubeflow-org-v1-tfjob"}
{"level":"info","ts":"2026-11-03T12:58:48.471374372Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"kubeflow.org/v1, Kind=TFJob","path":"/validate-kubeflow-org-v1-tfjob"}
{"level":"info","ts":"2026-11-03T12:58:48.471411029Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-kubeflow-org-v1-tfjob"}
{"level":"info","ts":"2026-11-03T12:58:48.471458188Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"kubeflow.org/v1, Kind=XGBoostJob","path":"/mutate-kubeflow-org-v1-xgboostjob"}
{"level":"info","ts":"2026-11-03T12:58:48.471489183Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-kubeflow-org-v1-xgboostjob"}
{"level":"info","ts":"2026-11-03T12:58:48.471510437Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"kubeflow.org/v1, Kind=XGBoostJob","path":"/validate-kubeflow-org-v1-xgboostjob"}
{"level":"info","ts":"2026-11-03T12:58:48.471541229Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-kubeflow-org-v1-xgboostjob"}
{"level":"info","ts":"2026-11-03T12:58:48.471586013Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"kubeflow.org/v2beta1, Kind=MPIJob","path":"/mutate-kubeflow-org-v2beta1-mpijob"}
{"level":"info","ts":"2026-11-03T12:58:48.471617564Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-kubeflow-org-v2beta1-mpijob"}
{"level":"info","ts":"2026-11-03T12:58:48.471637629Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"kubeflow.org/v2beta1, Kind=MPIJob","path":"/validate-kubeflow-org-v2beta1-mpijob"}
{"level":"info","ts":"2026-11-03T12:58:48.471667965Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-kubeflow-org-v2beta1-mpijob"}
{"level":"info","ts":"2026-11-03T12:58:48.471744017Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"/v1, Kind=Pod","path":"/mutate--v1-pod"}
{"level":"info","ts":"2026-11-03T12:58:48.471778947Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate--v1-pod"}
{"level":"info","ts":"2026-11-03T12:58:48.471798887Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"/v1, Kind=Pod","path":"/validate--v1-pod"}
{"level":"info","ts":"2026-11-03T12:58:48.471831651Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate--v1-pod"}
{"level":"info","ts":"2026-11-03T12:58:48.471879521Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"leaderworkerset.x-k8s.io/v1, Kind=LeaderWorkerSet","path":"/mutate-leaderworkerset-x-k8s-io-v1-leaderworkerset"}
{"level":"info","ts":"2026-11-03T12:58:48.471912606Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-leaderworkerset-x-k8s-io-v1-leaderworkerset"}
{"level":"info","ts":"2026-11-03T12:58:48.471932485Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"leaderworkerset.x-k8s.io/v1, Kind=LeaderWorkerSet","path":"/validate-leaderworkerset-x-k8s-io-v1-leaderworkerset"}
{"level":"info","ts":"2026-11-03T12:58:48.471964443Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-leaderworkerset-x-k8s-io-v1-leaderworkerset"}
{"level":"info","ts":"2026-11-03T12:58:48.472014564Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"ray.io/v1, Kind=RayCluster","path":"/mutate-ray-io-v1-raycluster"}
{"level":"info","ts":"2026-11-03T12:58:48.472047827Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-ray-io-v1-raycluster"}
{"level":"info","ts":"2026-11-03T12:58:48.472067947Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"ray.io/v1, Kind=RayCluster","path":"/validate-ray-io-v1-raycluster"}
{"level":"info","ts":"2026-11-03T12:58:48.472103196Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-ray-io-v1-raycluster"}
{"level":"info","ts":"2026-11-03T12:58:48.472172763Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"ray.io/v1, Kind=RayJob","path":"/mutate-ray-io-v1-rayjob"}
{"level":"info","ts":"2026-11-03T12:58:48.472208889Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-ray-io-v1-rayjob"}
{"level":"info","ts":"2026-11-03T12:58:48.472228257Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"ray.io/v1, Kind=RayJob","path":"/validate-ray-io-v1-rayjob"}
{"level":"info","ts":"2026-11-03T12:58:48.472261722Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-ray-io-v1-rayjob"}
{"level":"info","ts":"2026-11-03T12:58:48.472305213Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"ray.io/v1, Kind=RayService","path":"/mutate-ray-io-v1-rayservice"}
{"level":"info","ts":"2026-11-03T12:58:48.472549137Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-ray-io-v1-rayservice"}
{"level":"info","ts":"2026-11-03T12:58:48.472619986Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"ray.io/v1, Kind=RayService","path":"/validate-ray-io-v1-rayservice"}
{"level":"info","ts":"2026-11-03T12:58:48.472732714Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-ray-io-v1-rayservice"}
{"level":"info","ts":"2026-11-03T12:58:48.473520465Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"sparkoperator.k8s.io/v1beta2, Kind=SparkApplication","path":"/mutate-sparkoperator-k8s-io-v1beta2-sparkapplication"}
{"level":"info","ts":"2026-11-03T12:58:48.473696253Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-sparkoperator-k8s-io-v1beta2-sparkapplication"}
{"level":"info","ts":"2026-11-03T12:58:48.473796488Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"sparkoperator.k8s.io/v1beta2, Kind=SparkApplication","path":"/validate-sparkoperator-k8s-io-v1beta2-sparkapplication"}
{"level":"info","ts":"2026-11-03T12:58:48.473936862Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-sparkoperator-k8s-io-v1beta2-sparkapplication"}
{"level":"info","ts":"2026-11-03T12:58:48.474145372Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"apps/v1, Kind=StatefulSet","path":"/mutate-apps-v1-statefulset"}
{"level":"info","ts":"2026-11-03T12:58:48.474279113Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-apps-v1-statefulset"}
{"level":"info","ts":"2026-11-03T12:58:48.474498286Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"apps/v1, Kind=StatefulSet","path":"/validate-apps-v1-statefulset"}
{"level":"info","ts":"2026-11-03T12:58:48.474540787Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-apps-v1-statefulset"}
{"level":"info","ts":"2026-11-03T12:58:48.474946201Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","path":"/mutate-trainer-kubeflow-org-v1alpha1-trainjob"}
{"level":"info","ts":"2026-11-03T12:58:48.475004537Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-trainer-kubeflow-org-v1alpha1-trainjob"}
{"level":"info","ts":"2026-11-03T12:58:48.47502659Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","path":"/validate-trainer-kubeflow-org-v1alpha1-trainjob"}
{"level":"info","ts":"2026-11-03T12:58:48.475069034Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-trainer-kubeflow-org-v1alpha1-trainjob"}
{"level":"info","ts":"2026-11-03T12:58:48.475118758Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"kueue.x-k8s.io/v1beta2, Kind=Workload","path":"/mutate-kueue-x-k8s-io-v1beta2-workload"}
{"level":"info","ts":"2026-11-03T12:58:48.475178691Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-kueue-x-k8s-io-v1beta2-workload"}
{"level":"info","ts":"2026-11-03T12:58:48.47519999Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"kueue.x-k8s.io/v1beta2, Kind=Workload","path":"/validate-kueue-x-k8s-io-v1beta2-workload"}
{"level":"info","ts":"2026-11-03T12:58:48.475242174Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-kueue-x-k8s-io-v1beta2-workload"}
{"level":"info","ts":"2026-11-03T12:58:48.475320595Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/convert"}
{"level":"info","ts":"2026-11-03T12:58:48.475329626Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:319","msg":"Conversion webhook enabled","GVK":"kueue.x-k8s.io/v1beta2, Kind=Workload"}
{"level":"info","ts":"2026-11-03T12:58:48.475351842Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"kueue.x-k8s.io/v1beta2, Kind=ResourceFlavor","path":"/mutate-kueue-x-k8s-io-v1beta2-resourceflavor"}
{"level":"info","ts":"2026-11-03T12:58:48.475392178Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-kueue-x-k8s-io-v1beta2-resourceflavor"}
{"level":"info","ts":"2026-11-03T12:58:48.475412939Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"kueue.x-k8s.io/v1beta2, Kind=ResourceFlavor","path":"/validate-kueue-x-k8s-io-v1beta2-resourceflavor"}
{"level":"info","ts":"2026-11-03T12:58:48.475445337Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-kueue-x-k8s-io-v1beta2-resourceflavor"}
{"level":"info","ts":"2026-11-03T12:58:48.475496676Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"kueue.x-k8s.io/v1beta2, Kind=ClusterQueue","path":"/mutate-kueue-x-k8s-io-v1beta2-clusterqueue"}
{"level":"info","ts":"2026-11-03T12:58:48.475531408Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-kueue-x-k8s-io-v1beta2-clusterqueue"}
{"level":"info","ts":"2026-11-03T12:58:48.475558757Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"kueue.x-k8s.io/v1beta2, Kind=ClusterQueue","path":"/validate-kueue-x-k8s-io-v1beta2-clusterqueue"}
{"level":"info","ts":"2026-11-03T12:58:48.475592346Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-kueue-x-k8s-io-v1beta2-clusterqueue"}
{"level":"info","ts":"2026-11-03T12:58:48.475628471Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:319","msg":"Conversion webhook enabled","GVK":"kueue.x-k8s.io/v1beta2, Kind=ClusterQueue"}
{"level":"info","ts":"2026-11-03T12:58:48.475656134Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"kueue.x-k8s.io/v1beta2, Kind=Cohort","path":"/validate-kueue-x-k8s-io-v1beta2-cohort"}
{"level":"info","ts":"2026-11-03T12:58:48.475690096Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-kueue-x-k8s-io-v1beta2-cohort"}
{"level":"info","ts":"2026-11-03T12:58:48.475725664Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:319","msg":"Conversion webhook enabled","GVK":"kueue.x-k8s.io/v1beta2, Kind=Cohort"}
{"level":"info","ts":"2026-11-03T12:58:48.475757785Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:319","msg":"Conversion webhook enabled","GVK":"kueue.x-k8s.io/v1beta2, Kind=LocalQueue"}
{"level":"info","ts":"2026-11-03T12:58:48.475778264Z","logger":"setup","caller":"kueue/main.go:430","msg":"Starting manager"}
{"level":"info","ts":"2026-11-03T12:58:48.475843895Z","logger":"controller-runtime.metrics","caller":"server/server.go:208","msg":"Starting metrics server"}
{"level":"info","ts":"2026-11-03T12:58:48.475880201Z","caller":"manager/server.go:83","msg":"starting server","name":"health probe","addr":"[::]:8081"}
{"level":"info","ts":"2026-11-03T12:58:48.475939509Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:191","msg":"Starting webhook server"}
{"level":"info","ts":"2026-11-03T12:58:48.475971085Z","logger":"controller-runtime.metrics","caller":"server/server.go:247","msg":"Serving metrics server","bindAddress":":8443","secure":true}
{"level":"info","ts":"2026-11-03T12:58:48.476418791Z","logger":"controller-runtime.certwatcher","caller":"certwatcher/certwatcher.go:214","msg":"Updated current TLS certificate","cert":"/tmp/k8s-webhook-server/serving-certs/tls.crt","key":"/tmp/k8s-webhook-server/serving-certs/tls.key"}
{"level":"info","ts":"2026-11-03T12:58:48.47654278Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:242","msg":"Serving webhook server","host":"","port":9443}
{"level":"info","ts":"2026-11-03T12:58:48.476639963Z","logger":"controller-runtime.certwatcher","caller":"certwatcher/certwatcher.go:136","msg":"Starting certificate poll+watcher","cert":"/tmp/k8s-webhook-server/serving-certs/tls.crt","key":"/tmp/k8s-webhook-server/serving-certs/tls.key","interval":10}
{"level":"Level(-2)","ts":"2026-11-03T12:58:48.526991281Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1beta2.MultiKueueConfig","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-03T12:58:48.526992258Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1beta2.AdmissionCheck","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-03T12:58:48.527440945Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1beta2.MultiKueueCluster","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-03T12:58:48.527585755Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1.LimitRange","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-03T12:58:48.527711552Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1alpha1.TrainingRuntime","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-03T12:58:48.527944156Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1beta2.ResourceFlavor","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-03T12:58:48.528109865Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1beta2.LocalQueue","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-03T12:58:48.52819404Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1alpha1.ClusterTrainingRuntime","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-03T12:58:48.528344942Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1alpha1.TrainJob","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-03T12:58:48.52853894Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1.Node","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-03T12:58:48.528667893Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1beta2.Workload","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-03T12:58:48.674998906Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1.Pod","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"info","ts":"2026-11-03T12:58:48.676632775Z","logger":"controller-runtime.certwatcher","caller":"certwatcher/certwatcher.go:136","msg":"Starting certificate poll+watcher","cert":"/etc/kueue/metrics/certs/tls.crt","key":"/etc/kueue/metrics/certs/tls.key","interval":10}
{"level":"info","ts":"2026-11-03T12:58:48.67673081Z","logger":"admissioncheck-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"channel source: 0xb5026f7ae70"}
{"level":"info","ts":"2026-11-03T12:58:48.676790596Z","logger":"resourceflavor-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"channel source: 0xb5026f7acb0"}
{"level":"info","ts":"2026-11-03T12:58:48.676801944Z","logger":"admissioncheck-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1beta2.AdmissionCheck"}
{"level":"info","ts":"2026-11-03T12:58:48.676923974Z","logger":"resourceflavor-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1beta2.ResourceFlavor"}
{"level":"info","ts":"2026-11-03T12:58:48.676921809Z","logger":"workloadpriorityclass-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1beta2.WorkloadPriorityClass"}
{"level":"info","ts":"2026-11-03T12:58:48.677032632Z","logger":"localqueue-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1beta2.ClusterQueue"}
{"level":"info","ts":"2026-11-03T12:58:48.677058806Z","logger":"workload-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"channel source: 0xb5026f7b420"}
{"level":"info","ts":"2026-11-03T12:58:48.67707421Z","logger":"localqueue-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1beta2.LocalQueue"}
{"level":"info","ts":"2026-11-03T12:58:48.677075953Z","logger":"localqueue-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"channel source: 0xb5026f7afc0"}
{"level":"info","ts":"2026-11-03T12:58:48.677097914Z","logger":"workload-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1.LimitRange"}
{"level":"info","ts":"2026-11-03T12:58:48.677107411Z","logger":"workload-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1beta2.LocalQueue"}
{"level":"info","ts":"2026-11-03T12:58:48.677125683Z","logger":"workload-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1.RuntimeClass"}
{"level":"info","ts":"2026-11-03T12:58:48.677161973Z","logger":"workload-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1beta2.Workload"}
{"level":"info","ts":"2026-11-03T12:58:48.677162789Z","logger":"workload-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1beta2.ClusterQueue"}
{"level":"info","ts":"2026-11-03T12:58:48.677674556Z","logger":"cohort-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"channel source: 0xb5026f7b180"}
{"level":"info","ts":"2026-11-03T12:58:48.677720281Z","logger":"cohort-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1beta2.Cohort"}
{"level":"info","ts":"2026-11-03T12:58:48.67782164Z","logger":"tas-topology-controller","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1beta2.Topology"}
{"level":"info","ts":"2026-11-03T12:58:48.677881925Z","logger":"tas-topology-controller","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1beta2.ResourceFlavor"}
{"level":"info","ts":"2026-11-03T12:58:48.677951005Z","logger":"clusterqueue-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"channel source: 0xb5026f7b2d0"}
{"level":"info","ts":"2026-11-03T12:58:48.678260346Z","logger":"clusterqueue-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1.Namespace"}
{"level":"info","ts":"2026-11-03T12:58:48.678533255Z","logger":"tas-resource-flavor-controller","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"channel source: 0xb5026fe1420"}
{"level":"info","ts":"2026-11-03T12:58:48.678631796Z","logger":"tas-node-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1.Node"}
{"level":"info","ts":"2026-11-03T12:58:48.679013555Z","logger":"tas-node-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1.Pod"}
{"level":"info","ts":"2026-11-03T12:58:48.679022643Z","logger":"tas-topology-ungater","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1beta2.Workload"}
{"level":"info","ts":"2026-11-03T12:58:48.679541785Z","logger":"tas-resource-flavor-controller","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1beta2.ResourceFlavor"}
{"level":"info","ts":"2026-11-03T12:58:48.679671855Z","logger":"tas-non-tas-usage-controller","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1.Pod"}
{"level":"info","ts":"2026-11-03T12:58:48.679904929Z","logger":"tas-topology-ungater","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1.Pod"}
{"level":"info","ts":"2026-11-03T12:58:48.680000137Z","logger":"ElasticJobUngater","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1beta2.Workload"}
{"level":"info","ts":"2026-11-03T12:58:48.680354271Z","logger":"ElasticJobUngater","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1.Pod"}
I1103 12:58:48.680466       1 leaderelection.go:258] "Attempting to acquire leader lease..." lock="openshift-kueue-operator/c1f6bfd2.kueue.x-k8s.io"
{"level":"info","ts":"2026-11-03T12:58:48.680639444Z","logger":"clusterqueue-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1beta2.ClusterQueue"}
{"level":"Level(-2)","ts":"2026-11-03T12:58:48.734804654Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1beta2.WorkloadPriorityClass","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
I1103 12:58:48.736689       1 plugins.go:157] Loaded 1 mutating admission controller(s) successfully in the following order: NamespaceLifecycle.
I1103 12:58:48.739546       1 handler.go:304] Adding GroupVersion visibility.kueue.x-k8s.io v1beta2 to ResourceManager
I1103 12:58:48.739742       1 handler.go:304] Adding GroupVersion visibility.kueue.x-k8s.io v1beta1 to ResourceManager
{"level":"Level(-2)","ts":"2026-11-03T12:58:48.776382219Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1.RuntimeClass","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"info","ts":"2026-11-03T12:58:48.77715123Z","logger":"admissioncheck-reconciler","caller":"controller/controller.go:302","msg":"Starting Controller","replica-role":"follower"}
{"level":"info","ts":"2026-11-03T12:58:48.77715577Z","logger":"resourceflavor-reconciler","caller":"controller/controller.go:302","msg":"Starting Controller","replica-role":"follower"}
{"level":"info","ts":"2026-11-03T12:58:48.777201175Z","logger":"admissioncheck-reconciler","caller":"controller/controller.go:305","msg":"Starting workers","replica-role":"follower","worker count":1}
{"level":"info","ts":"2026-11-03T12:58:48.777209811Z","logger":"resourceflavor-reconciler","caller":"controller/controller.go:305","msg":"Starting workers","replica-role":"follower","worker count":1}
{"level":"Level(-2)","ts":"2026-11-03T12:58:48.778747679Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1.Namespace","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
I1103 12:58:48.792632       1 requestheader_controller.go:180] Starting RequestHeaderAuthRequestController
I1103 12:58:48.792649       1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::client-ca-file"
I1103 12:58:48.792656       1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file"
I1103 12:58:48.792667       1 shared_informer.go:402] "Waiting for caches to sync"
I1103 12:58:48.792671       1 shared_informer.go:402] "Waiting for caches to sync"
I1103 12:58:48.792661       1 shared_informer.go:402] "Waiting for caches to sync"
I1103 12:58:48.793092       1 dynamic_serving_content.go:135] "Starting controller" name="serving-cert::/visibility/tls.crt::/visibility/tls.key"
I1103 12:58:48.793522       1 secure_serving.go:214] Serving securely on [::]:8082
I1103 12:58:48.793553       1 tlsconfig.go:243] "Starting DynamicServingCertificateController"
I1103 12:58:48.793602       1 apf_controller.go:377] Starting API Priority and Fairness config controller
{"level":"Level(-2)","ts":"2026-11-03T12:58:48.857105377Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1beta2.ClusterQueue","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
I1103 12:58:48.893463       1 shared_informer.go:409] "Caches are synced"
I1103 12:58:48.893475       1 shared_informer.go:409] "Caches are synced"
I1103 12:58:48.893490       1 shared_informer.go:409] "Caches are synced"
I1103 12:58:48.893706       1 apf_controller.go:382] Running API Priority and Fairness config worker
I1103 12:58:48.893716       1 apf_controller.go:385] Running API Priority and Fairness periodic rebalancing process
{"level":"Level(-2)","ts":"2026-11-03T12:58:48.894473913Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1beta2.Topology","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-03T12:58:49.883105457Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1beta2.Cohort","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"info","ts":"2026-11-03T12:58:49.978404612Z","logger":"workloadpriorityclass-reconciler","caller":"controller/controller.go:302","msg":"Starting Controller","replica-role":"follower"}
{"level":"info","ts":"2026-11-03T12:58:49.978429562Z","logger":"localqueue-reconciler","caller":"controller/controller.go:302","msg":"Starting Controller","replica-role":"follower"}
{"level":"info","ts":"2026-11-03T12:58:49.978462064Z","logger":"workloadpriorityclass-reconciler","caller":"controller/controller.go:305","msg":"Starting workers","replica-role":"follower","worker count":1}
{"level":"info","ts":"2026-11-03T12:58:49.978465861Z","logger":"localqueue-reconciler","caller":"controller/controller.go:305","msg":"Starting workers","replica-role":"follower","worker count":1}
{"level":"info","ts":"2026-11-03T12:58:49.980604335Z","logger":"tas-topology-ungater","caller":"controller/controller.go:302","msg":"Starting Controller","replica-role":"follower"}
{"level":"info","ts":"2026-11-03T12:58:49.980640207Z","logger":"tas-topology-ungater","caller":"controller/controller.go:305","msg":"Starting workers","replica-role":"follower","worker count":5}
{"level":"info","ts":"2026-11-03T12:58:49.980634256Z","logger":"tas-topology-controller","caller":"controller/controller.go:302","msg":"Starting Controller","replica-role":"follower"}
{"level":"info","ts":"2026-11-03T12:58:49.980656407Z","logger":"tas-topology-controller","caller":"controller/controller.go:305","msg":"Starting workers","replica-role":"follower","worker count":1}
{"level":"info","ts":"2026-11-03T12:58:49.98171528Z","logger":"tas-non-tas-usage-controller","caller":"controller/controller.go:302","msg":"Starting Controller","replica-role":"follower"}
{"level":"info","ts":"2026-11-03T12:58:49.981741648Z","logger":"tas-non-tas-usage-controller","caller":"controller/controller.go:305","msg":"Starting workers","replica-role":"follower","worker count":5}
{"level":"info","ts":"2026-11-03T12:58:49.981727642Z","logger":"ElasticJobUngater","caller":"controller/controller.go:302","msg":"Starting Controller","replica-role":"follower"}
{"level":"info","ts":"2026-11-03T12:58:49.981758789Z","logger":"ElasticJobUngater","caller":"controller/controller.go:305","msg":"Starting workers","replica-role":"follower","worker count":5}
{"level":"info","ts":"2026-11-03T12:58:49.981758887Z","logger":"tas-resource-flavor-controller","caller":"controller/controller.go:302","msg":"Starting Controller","replica-role":"follower"}
{"level":"info","ts":"2026-11-03T12:58:49.981765471Z","logger":"tas-node-reconciler","caller":"controller/controller.go:302","msg":"Starting Controller","replica-role":"follower"}
{"level":"info","ts":"2026-11-03T12:58:49.981779122Z","logger":"tas-resource-flavor-controller","caller":"controller/controller.go:305","msg":"Starting workers","replica-role":"follower","worker count":1}
{"level":"info","ts":"2026-11-03T12:58:49.981790407Z","logger":"tas-node-reconciler","caller":"controller/controller.go:305","msg":"Starting workers","replica-role":"follower","worker count":1}
{"level":"info","ts":"2026-11-03T12:58:49.981782321Z","logger":"workload-reconciler","caller":"controller/controller.go:302","msg":"Starting Controller","replica-role":"follower"}
{"level":"info","ts":"2026-11-03T12:58:49.981805339Z","logger":"workload-reconciler","caller":"controller/controller.go:305","msg":"Starting workers","replica-role":"follower","worker count":5}
{"level":"info","ts":"2026-11-03T12:58:49.981805659Z","logger":"clusterqueue-reconciler","caller":"controller/controller.go:302","msg":"Starting Controller","replica-role":"follower"}
{"level":"info","ts":"2026-11-03T12:58:49.981826008Z","logger":"clusterqueue-reconciler","caller":"controller/controller.go:305","msg":"Starting workers","replica-role":"follower","worker count":1}
{"level":"info","ts":"2026-11-03T12:58:50.078577002Z","logger":"cohort-reconciler","caller":"controller/controller.go:302","msg":"Starting Controller","replica-role":"follower"}
{"level":"info","ts":"2026-11-03T12:58:50.078608508Z","logger":"cohort-reconciler","caller":"controller/controller.go:305","msg":"Starting workers","replica-role":"follower","worker count":1}
{"level":"Level(-2)","ts":"2026-11-03T13:24:58.477654442Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:141","msg":"ResourceFlavor create event","replica-role":"follower","resourceFlavor":{"name":"rf-rtfks"}}
{"level":"Level(-2)","ts":"2026-11-03T13:24:58.821755011Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:351","msg":"ClusterQueue create event","replica-role":"follower","clusterQueue":{"name":"cq-gh466"}}
{"level":"Level(-2)","ts":"2026-11-03T13:24:58.840945365Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:240","msg":"LocalQueue create event","replica-role":"follower","localQueue":{"name":"lq-5s27t","namespace":"test-ns-56kxh"}}
{"level":"Level(-2)","ts":"2026-11-03T13:24:58.881112544Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"follower","clusterQueue":{"name":"cq-gh466"}}
{"level":"Level(-2)","ts":"2026-11-03T13:24:58.897002252Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"follower","localQueue":{"name":"lq-5s27t","namespace":"test-ns-56kxh"}}
{"level":"Level(-2)","ts":"2026-11-03T13:24:58.955297403Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"follower","localQueue":{"name":"lq-5s27t","namespace":"test-ns-56kxh"}}
{"level":"Level(-2)","ts":"2026-11-03T13:25:00.178727435Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1276","msg":"Workload create event","replica-role":"follower","workload":{"name":"pod-dw-deployment-8qgpj-cbfb688fc-n9sv6-0947e","namespace":"test-ns-56kxh"},"queue":"lq-5s27t","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-03T13:25:00.344319646Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"follower","workload":{"name":"pod-dw-deployment-8qgpj-cbfb688fc-n9sv6-0947e","namespace":"test-ns-56kxh"},"queue":"lq-5s27t","status":"admitted","prevStatus":"pending","clusterQueue":"cq-gh466"}
{"level":"Level(-2)","ts":"2026-11-03T13:25:01.288880892Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"follower","localQueue":{"name":"lq-5s27t","namespace":"test-ns-56kxh"}}
{"level":"Level(-2)","ts":"2026-11-03T13:25:01.291591816Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"follower","clusterQueue":{"name":"cq-gh466"}}
{"level":"info","ts":"2026-11-03T13:25:14.570449948Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:117","msg":"Validating create","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"d240fbd6a54b","namespace":"test-ns-56kxh"},"namespace":"test-ns-56kxh","name":"d240fbd6a54b","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:redhat-ods-applications:trainer-e2e","requestID":"a7fbed60-76d6-455d-82c6-73291fdbf57d","replica-role":"follower"}
{"level":"Level(-2)","ts":"2026-11-03T13:25:14.957238313Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1276","msg":"Workload create event","replica-role":"follower","workload":{"name":"trainjob-d240fbd6a54b-429f8","namespace":"test-ns-56kxh"},"queue":"lq-5s27t","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-03T13:25:15.068882474Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"follower","workload":{"name":"trainjob-d240fbd6a54b-429f8","namespace":"test-ns-56kxh"},"queue":"lq-5s27t","status":"admitted","prevStatus":"pending","clusterQueue":"cq-gh466"}
{"level":"info","ts":"2026-11-03T13:25:15.807520358Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"d240fbd6a54b","namespace":"test-ns-56kxh"},"namespace":"test-ns-56kxh","name":"d240fbd6a54b","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"691aef86-69c6-47d0-85f1-98ba06c0d81b","replica-role":"follower"}
{"level":"info","ts":"2026-11-03T13:25:15.972347646Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"d240fbd6a54b","namespace":"test-ns-56kxh"},"namespace":"test-ns-56kxh","name":"d240fbd6a54b","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"60a1c5c9-636d-43a7-b612-443b1595f110","replica-role":"follower"}
{"level":"Level(-2)","ts":"2026-11-03T13:25:16.016318438Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"follower","localQueue":{"name":"lq-5s27t","namespace":"test-ns-56kxh"}}
{"level":"Level(-2)","ts":"2026-11-03T13:25:16.019160661Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"follower","clusterQueue":{"name":"cq-gh466"}}
{"level":"Level(-2)","ts":"2026-11-03T13:25:16.428521823Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1.PartialObjectMetadata","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-03T13:25:16.481089731Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1.PartialObjectMetadata","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-03T13:25:49.811664359Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"follower","workload":{"name":"trainjob-d240fbd6a54b-429f8","namespace":"test-ns-56kxh"},"queue":"lq-5s27t","status":"finished","prevStatus":"admitted","clusterQueue":"cq-gh466"}
{"level":"Level(-2)","ts":"2026-11-03T13:25:50.082614348Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"follower","workload":{"name":"trainjob-d240fbd6a54b-429f8","namespace":"test-ns-56kxh"},"queue":"lq-5s27t","status":"finished","clusterQueue":"cq-gh466"}
{"level":"Level(-2)","ts":"2026-11-03T13:25:50.871877212Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"follower","localQueue":{"name":"lq-5s27t","namespace":"test-ns-56kxh"}}
{"level":"Level(-2)","ts":"2026-11-03T13:25:50.874975207Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"follower","clusterQueue":{"name":"cq-gh466"}}
{"level":"Level(-2)","ts":"2026-11-03T13:25:52.372359444Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1312","msg":"Workload delete event","replica-role":"follower","workload":{"name":"trainjob-d240fbd6a54b-429f8","namespace":"test-ns-56kxh"},"queue":"lq-5s27t","status":"finished"}
{"level":"Level(-2)","ts":"2026-11-03T13:25:53.68747722Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"follower","clusterQueue":{"name":"cq-gh466"}}
{"level":"Level(-2)","ts":"2026-11-03T13:25:53.704693849Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:171","msg":"ResourceFlavor update event","replica-role":"follower","resourceFlavor":{"name":"rf-rtfks"}}
{"level":"Level(-2)","ts":"2026-11-03T13:25:53.746800241Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"follower","clusterQueue":{"name":"cq-gh466"}}
{"level":"Level(-2)","ts":"2026-11-03T13:25:53.803257385Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"follower","localQueue":{"name":"lq-5s27t","namespace":"test-ns-56kxh"}}
{"level":"Level(-2)","ts":"2026-11-03T13:25:54.060080339Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:141","msg":"ResourceFlavor create event","replica-role":"follower","resourceFlavor":{"name":"rf-cdk49"}}
{"level":"Level(-2)","ts":"2026-11-03T13:25:54.178394462Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:351","msg":"ClusterQueue create event","replica-role":"follower","clusterQueue":{"name":"cq-mdlnd"}}
{"level":"Level(-2)","ts":"2026-11-03T13:25:54.196008086Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:240","msg":"LocalQueue create event","replica-role":"follower","localQueue":{"name":"lq-xd7dm","namespace":"test-ns-m2fzh"}}
{"level":"Level(-2)","ts":"2026-11-03T13:25:54.236840959Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"follower","clusterQueue":{"name":"cq-mdlnd"}}
{"level":"Level(-2)","ts":"2026-11-03T13:25:54.251807714Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"follower","localQueue":{"name":"lq-xd7dm","namespace":"test-ns-m2fzh"}}
{"level":"Level(-2)","ts":"2026-11-03T13:25:54.30786093Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"follower","localQueue":{"name":"lq-xd7dm","namespace":"test-ns-m2fzh"}}
{"level":"info","ts":"2026-11-03T13:25:54.311422555Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:117","msg":"Validating create","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-preemption-trainjob-znhq5","namespace":"test-ns-m2fzh"},"namespace":"test-ns-m2fzh","name":"test-preemption-trainjob-znhq5","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:admin","requestID":"5034c546-49ff-45d9-b7c4-be5afc6a5ac2","replica-role":"follower"}
{"level":"Level(-2)","ts":"2026-11-03T13:25:54.673222625Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1276","msg":"Workload create event","replica-role":"follower","workload":{"name":"trainjob-test-preemption-trainjob-znhq5-b345a","namespace":"test-ns-m2fzh"},"queue":"lq-xd7dm","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-03T13:25:54.780848208Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"follower","workload":{"name":"trainjob-test-preemption-trainjob-znhq5-b345a","namespace":"test-ns-m2fzh"},"queue":"lq-xd7dm","status":"admitted","prevStatus":"pending","clusterQueue":"cq-mdlnd"}
{"level":"info","ts":"2026-11-03T13:25:55.347287438Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-preemption-trainjob-znhq5","namespace":"test-ns-m2fzh"},"namespace":"test-ns-m2fzh","name":"test-preemption-trainjob-znhq5","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"8012f092-712a-432a-9f1b-f859331f4bea","replica-role":"follower"}
{"level":"info","ts":"2026-11-03T13:25:55.505855791Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-preemption-trainjob-znhq5","namespace":"test-ns-m2fzh"},"namespace":"test-ns-m2fzh","name":"test-preemption-trainjob-znhq5","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"6acdfe74-8e4e-4ee5-85f1-0e33f49f3f08","replica-role":"follower"}
{"level":"Level(-2)","ts":"2026-11-03T13:25:55.732614587Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"follower","localQueue":{"name":"lq-xd7dm","namespace":"test-ns-m2fzh"}}
{"level":"Level(-2)","ts":"2026-11-03T13:25:55.73535708Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"follower","clusterQueue":{"name":"cq-mdlnd"}}
{"level":"Level(-2)","ts":"2026-11-03T13:25:57.616822358Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"follower","workload":{"name":"trainjob-test-preemption-trainjob-znhq5-b345a","namespace":"test-ns-m2fzh"},"queue":"lq-xd7dm","status":"admitted","clusterQueue":"cq-mdlnd"}
{"level":"Level(-2)","ts":"2026-11-03T13:25:57.616884215Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1381","msg":"Workload will not be queued because the workload is not active","replica-role":"follower","workload":{"name":"trainjob-test-preemption-trainjob-znhq5-b345a","namespace":"test-ns-m2fzh"},"queue":"lq-xd7dm","status":"admitted","clusterQueue":"cq-mdlnd"}
{"level":"Level(-2)","ts":"2026-11-03T13:25:57.72826239Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"follower","workload":{"name":"trainjob-test-preemption-trainjob-znhq5-b345a","namespace":"test-ns-m2fzh"},"queue":"lq-xd7dm","status":"admitted","clusterQueue":"cq-mdlnd"}
{"level":"Level(-2)","ts":"2026-11-03T13:25:57.728321451Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1381","msg":"Workload will not be queued because the workload is not active","replica-role":"follower","workload":{"name":"trainjob-test-preemption-trainjob-znhq5-b345a","namespace":"test-ns-m2fzh"},"queue":"lq-xd7dm","status":"admitted","clusterQueue":"cq-mdlnd"}
{"level":"info","ts":"2026-11-03T13:25:58.030344588Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-preemption-trainjob-znhq5","namespace":"test-ns-m2fzh"},"namespace":"test-ns-m2fzh","name":"test-preemption-trainjob-znhq5","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"d3ddff88-cc49-45de-b60f-48f38f839582","replica-role":"follower"}
{"level":"Level(-2)","ts":"2026-11-03T13:25:58.67559284Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"follower","localQueue":{"name":"lq-xd7dm","namespace":"test-ns-m2fzh"}}
{"level":"Level(-2)","ts":"2026-11-03T13:25:58.6764772Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"follower","clusterQueue":{"name":"cq-mdlnd"}}
{"level":"Level(-2)","ts":"2026-11-03T13:25:58.686827324Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"follower","clusterQueue":{"name":"cq-mdlnd"}}
{"level":"Level(-2)","ts":"2026-11-03T13:25:58.704050556Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:171","msg":"ResourceFlavor update event","replica-role":"follower","resourceFlavor":{"name":"rf-cdk49"}}
{"level":"Level(-2)","ts":"2026-11-03T13:25:58.794340949Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:376","msg":"ClusterQueue delete event","replica-role":"follower","clusterQueue":{"name":"cq-mdlnd"}}
{"level":"Level(-2)","ts":"2026-11-03T13:25:58.794422079Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:385","msg":"Cleared resource metrics for deleted ClusterQueue.","replica-role":"follower","clusterQueue":{"name":"cq-mdlnd"}}
{"level":"Level(-2)","ts":"2026-11-03T13:25:58.849654194Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"follower","localQueue":{"name":"lq-xd7dm","namespace":"test-ns-m2fzh"}}
{"level":"Level(-2)","ts":"2026-11-03T13:25:58.901557503Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:159","msg":"ResourceFlavor delete event","replica-role":"follower","resourceFlavor":{"name":"rf-cdk49"}}
{"level":"info","ts":"2026-11-03T13:25:58.974402181Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-preemption-trainjob-znhq5","namespace":"test-ns-m2fzh"},"namespace":"test-ns-m2fzh","name":"test-preemption-trainjob-znhq5","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"e395492e-7d80-4dca-a51e-682cb354e5e7","replica-role":"follower"}
{"level":"Level(-2)","ts":"2026-11-03T13:25:59.012658831Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:141","msg":"ResourceFlavor create event","replica-role":"follower","resourceFlavor":{"name":"rf-8z72w"}}
{"level":"Level(-2)","ts":"2026-11-03T13:25:59.129414052Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:351","msg":"ClusterQueue create event","replica-role":"follower","clusterQueue":{"name":"cq-5b2m5"}}
{"level":"Level(-2)","ts":"2026-11-03T13:25:59.135489487Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"follower","workload":{"name":"trainjob-test-preemption-trainjob-znhq5-b345a","namespace":"test-ns-m2fzh"},"queue":"lq-xd7dm","status":"pending","prevStatus":"admitted","prevClusterQueue":"cq-mdlnd"}
{"level":"Level(-2)","ts":"2026-11-03T13:25:59.135535527Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1381","msg":"Workload will not be queued because the workload is not active","replica-role":"follower","workload":{"name":"trainjob-test-preemption-trainjob-znhq5-b345a","namespace":"test-ns-m2fzh"},"queue":"lq-xd7dm","status":"pending","prevStatus":"admitted","prevClusterQueue":"cq-mdlnd"}
{"level":"Level(-2)","ts":"2026-11-03T13:25:59.147038732Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:240","msg":"LocalQueue create event","replica-role":"follower","localQueue":{"name":"lq-rkh4q","namespace":"test-ns-rsk86"}}
{"level":"Level(-2)","ts":"2026-11-03T13:25:59.185562273Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"follower","clusterQueue":{"name":"cq-5b2m5"}}
{"level":"Level(-2)","ts":"2026-11-03T13:25:59.204662179Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"follower","localQueue":{"name":"lq-rkh4q","namespace":"test-ns-rsk86"}}
{"level":"Level(-2)","ts":"2026-11-03T13:25:59.246764023Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"follower","workload":{"name":"trainjob-test-preemption-trainjob-znhq5-b345a","namespace":"test-ns-m2fzh"},"queue":"lq-xd7dm","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-03T13:25:59.24682315Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1381","msg":"Workload will not be queued because the workload is not active","replica-role":"follower","workload":{"name":"trainjob-test-preemption-trainjob-znhq5-b345a","namespace":"test-ns-m2fzh"},"queue":"lq-xd7dm","status":"pending"}
{"level":"info","ts":"2026-11-03T13:25:59.260028001Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:117","msg":"Validating create","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-runtimepatches-trainjob-nzb86","namespace":"test-ns-rsk86"},"namespace":"test-ns-rsk86","name":"test-runtimepatches-trainjob-nzb86","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:admin","requestID":"51259d6a-248d-452f-a188-641ec7d63b86","replica-role":"follower"}
{"level":"Level(-2)","ts":"2026-11-03T13:25:59.262611686Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"follower","localQueue":{"name":"lq-rkh4q","namespace":"test-ns-rsk86"}}
{"level":"Level(-2)","ts":"2026-11-03T13:25:59.624326357Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1276","msg":"Workload create event","replica-role":"follower","workload":{"name":"trainjob-test-runtimepatches-trainjob-nzb86-28b7a","namespace":"test-ns-rsk86"},"queue":"lq-rkh4q","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-03T13:25:59.73293493Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"follower","workload":{"name":"trainjob-test-runtimepatches-trainjob-nzb86-28b7a","namespace":"test-ns-rsk86"},"queue":"lq-rkh4q","status":"admitted","prevStatus":"pending","clusterQueue":"cq-5b2m5"}
{"level":"info","ts":"2026-11-03T13:26:00.155766061Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-runtimepatches-trainjob-nzb86","namespace":"test-ns-rsk86"},"namespace":"test-ns-rsk86","name":"test-runtimepatches-trainjob-nzb86","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"b20a3045-6527-48c6-910a-4b2e87102c2e","replica-role":"follower"}
{"level":"Level(-2)","ts":"2026-11-03T13:26:00.398465321Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"follower","clusterQueue":{"name":"cq-5b2m5"}}
{"level":"Level(-2)","ts":"2026-11-03T13:26:00.416210084Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:171","msg":"ResourceFlavor update event","replica-role":"follower","resourceFlavor":{"name":"rf-8z72w"}}
{"level":"Level(-2)","ts":"2026-11-03T13:26:00.465201419Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"follower","clusterQueue":{"name":"cq-5b2m5"}}
{"level":"Level(-2)","ts":"2026-11-03T13:26:00.521554788Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"follower","localQueue":{"name":"lq-rkh4q","namespace":"test-ns-rsk86"}}
{"level":"info","ts":"2026-11-03T13:26:00.523436085Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-runtimepatches-trainjob-nzb86","namespace":"test-ns-rsk86"},"namespace":"test-ns-rsk86","name":"test-runtimepatches-trainjob-nzb86","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"53c5fdb8-c273-4ed5-8cad-6cf9411cee5e","replica-role":"follower"}
{"level":"info","ts":"2026-11-03T13:26:00.685810443Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-runtimepatches-trainjob-nzb86","namespace":"test-ns-rsk86"},"namespace":"test-ns-rsk86","name":"test-runtimepatches-trainjob-nzb86","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"47fc8735-c799-4ec0-a180-0918808423fe","replica-role":"follower"}