{"level":"info","ts":"2026-11-03T03:32:12.785312595Z","logger":"setup","caller":"kueue/main.go:721","msg":"Configuration loaded","config":"apiVersion: config.kueue.x-k8s.io/v1beta2\nclientConnection:\n  burst: 100\n  qps: 50\ncontroller:\n  groupKindConcurrency:\n    ClusterQueue.kueue.x-k8s.io: 1\n    Job.batch: 5\n    LocalQueue.kueue.x-k8s.io: 1\n    Pod: 5\n    ResourceFlavor.kueue.x-k8s.io: 1\n    Workload.kueue.x-k8s.io: 5\nhealth:\n  healthProbeBindAddress: :8081\nintegrations:\n  frameworks:\n  - trainer.kubeflow.org/trainjob\n  - deployment\ninternalCertManagement:\n  enable: false\nkind: Configuration\nleaderElection:\n  leaderElect: true\n  leaseDuration: 2m17s\n  renewDeadline: 1m47s\n  resourceLock: leases\n  resourceName: c1f6bfd2.kueue.x-k8s.io\n  resourceNamespace: \"\"\n  retryPeriod: 26s\nmanageJobsWithoutQueueName: false\nmanagedJobsNamespaceSelector:\n  matchLabels:\n    kueue.openshift.io/managed: \"true\"\nmetrics:\n  bindAddress: :8443\n  enableClusterQueueResources: true\n  localQueueMetrics:\n    enable: true\nmultiKueue:\n  dispatcherName: kueue.x-k8s.io/multikueue-dispatcher-all-at-once\n  gcInterval: 1m0s\n  origin: multikueue\n  workerLostTimeout: 15m0s\nnamespace: openshift-kueue-operator\ntls:\n  cipherSuites:\n  - TLS_AES_128_GCM_SHA256\n  - TLS_AES_256_GCM_SHA384\n  - TLS_CHACHA20_POLY1305_SHA256\n  - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256\n  - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256\n  - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384\n  - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\n  - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256\n  - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256\n  minVersion: VersionTLS12\nvisibilityServer:\n  bindPort: 8082\nwebhook:\n  certDir: /tmp/k8s-webhook-server/serving-certs\n  port: 9443\n"}
{"level":"info","ts":"2026-11-03T03:32:12.785508963Z","logger":"setup","caller":"kueue/main.go:170","msg":"Initializing","gitVersion":"d12900d9-dirty","gitCommit":"d12900d94cd6813a3882a6391ffd70009a4b61d9","buildDate":"2026-10-05T16:41:45Z"}
{"level":"Level(-2)","ts":"2026-11-03T03:32:12.78589247Z","logger":"setup","caller":"features/kube_features.go:1144","msg":"Loaded feature gates","featureGates":{"AdmissionFairSharing":true,"AdmissionGatedBy":false,"AssignQueueLabelsForPods":true,"CleanupProvisioningRequestsOnEviction":true,"ConcurrentAdmission":false,"CustomMetricLabels":false,"DRAExtendedResources":false,"DeferRayServiceFinalizationForRedisCleanup":true,"DeploymentJobUIDLabel":false,"DynamicResourceAllocation":false,"ElasticJobsViaWorkloadSlices":true,"ElasticJobsViaWorkloadSlicesWithTAS":false,"EnforceProvisioningPodTemplateContents":false,"FailureRecoveryPolicy":false,"FairSharingPreemptWithinNominal":true,"FairSharingPrioritizeNonBorrowing":true,"FairSharingReevaluatePreemptionCandidates":false,"FastQuotaReleaseInPodIntegration":false,"FinalizeTerminatingPodGroups":false,"FinishOrphanedWorkloads":true,"FlavorFungibility":true,"FlavorFungibilityPreserveScanProgress":true,"HierarchicalCohorts":true,"HighMaxParallelismWithinReconcile":false,"KueueDRAIntegration":true,"KueueDRAIntegrationExtendedResource":false,"KueueDRAIntegrationPartitionableDevices":false,"KueueDRARejectWorkloadsWhenDRADisabled":true,"LWSImmutableGroupSize":true,"LendingLimit":true,"LocalQueueDefaulting":true,"LocalQueueMetrics":true,"ManagedJobsNamespaceSelectorAlwaysRespected":true,"MetricForWorkloadCreationLatency":true,"MetricsForCohorts":true,"MultiKueue":true,"MultiKueueAdaptersForCustomJobs":true,"MultiKueueAllowInsecureKubeconfigs":false,"MultiKueueBatchJobClearingTTLSecondsAfterFinishedOnWorkerCluster":false,"MultiKueueBatchJobWithManagedBy":true,"MultiKueueClusterProfile":false,"MultiKueueManagerQuotaAutomation":false,"MultiKueueOrchestratedPreemption":false,"MultiKueueRedoAdmissionOnEvictionInWorker":true,"MultiKueueRemoteSpecSync":true,"MultiKueueReuseClientConnectionConfigForWorkers":false,"MultiKueueWaitForWorkloadAdmitted":true,"ObjectRetentionPolicies":true,"PartialAdmission":true,"PodGroupSchedulingShapeOrdering":false,"PodIntegrationCountSucceededPodsAsReady":false,"PodIntegrationValidateGroupOwner":true,"PrioritizePreemptorWorkloads":false,"PriorityBoost":false,"PrioritySortingWithinCohort":true,"PropagateBatchJobLabelsToWorkload":true,"QuotaCheckStrategy":false,"ReclaimablePods":true,"RecomputeAssignmentUponPreemptionTargetsOverlap":false,"RejectUpdatesToCQWithInvalidOnFlavors":false,"RemoveFinalizersWithStrictPatch":true,"ReservedResourceNameValidation":false,"SanitizePodSets":true,"SchedulerLongRequeueInterval":false,"SchedulerTimestampPreemptionBuffer":false,"SchedulingEquivalenceHashing":true,"SchedulingEquivalenceHashingIgnorePodSetName":false,"ShortWorkloadNames":false,"SkipAncestorCheckForDeletedWorkloads":true,"SkipChildJobSuspension":false,"SkipFinalizersForPodsSuspendedByParent":true,"SkipReassignmentForPodOwnedWorkloads":false,"SparkApplicationIntegration":false,"TASBalancedPlacement":false,"TASCacheNodeMatchResults":true,"TASCacheTopologyTree":false,"TASCachingRemainingResources":true,"TASFailedNodeReplacement":true,"TASFailedNodeReplacementFailFast":true,"TASGroupedPodSetSlicing":false,"TASHandleOverlappingFlavors":true,"TASMultiLayerTopology":false,"TASPartialSlices":true,"TASProfileMixed":true,"TASRecomputeAssignmentWithinSchedulingCycle":true,"TASRejectFalseUnconstrainedTopology":false,"TASReplaceNodeDueToNotReadyOverFixedTime":true,"TASReplaceNodeOnNodeTaints":true,"TASReplaceNodeOnPodTermination":true,"TASRespectNodeAffinityPreferred":false,"TASValidateWorkloadSliceSize":true,"TLSOptions":true,"TopologyAwareScheduling":true,"UnadmittedWorkloadsExplicitStatus":false,"UnadmittedWorkloadsObservability":false,"ValidateRayAndSparkJobUpdates":true,"VectorizedResourceRequests":true,"VisibilityOnDemand":true,"WorkloadIdentifierAnnotations":true,"WorkloadPriorityClassDefaulting":false,"WorkloadRequestUseMergePatch":false,"WorkloadValidateResourcesAreNonNegative":true,"WorkloadValidationForPodSetMetadata":true}}
{"level":"info","ts":"2026-11-03T03:32:12.78597205Z","logger":"setup","caller":"kueue/main.go:213","msg":"Initializing metrics certificate watcher using provided certificates","metrics-cert-path":"/etc/kueue/metrics/certs"}
{"level":"info","ts":"2026-11-03T03:32:12.786249984Z","logger":"controller-runtime.certwatcher","caller":"certwatcher/certwatcher.go:214","msg":"Updated current TLS certificate","cert":"/etc/kueue/metrics/certs/tls.crt","key":"/etc/kueue/metrics/certs/tls.key"}
{"level":"Level(-2)","ts":"2026-11-03T03:32:12.786776666Z","logger":"setup","caller":"kueue/main.go:257","msg":"K8S Client","qps":50,"burst":100}
{"level":"info","ts":"2026-11-03T03:32:13.08656347Z","logger":"setup","caller":"kueue/main.go:683","msg":"RoleTracker: leader election enabled"}
{"level":"error","ts":"2026-11-03T03:32:13.138059822Z","logger":"setup","caller":"kueue/main.go:445","msg":"Skipping admission check controller setup: Provisioning Requests not supported (Possible cause: missing or unsupported cluster-autoscaler)","error":"no matches for kind \"ProvisioningRequest\" in version \"autoscaling.x-k8s.io/v1\"","stacktrace":"main.setupIndexes\n\t/workspace/upstream/kueue/src/cmd/kueue/main.go:445\nmain.main\n\t/workspace/upstream/kueue/src/cmd/kueue/main.go:363\nruntime.main\n\t/usr/lib/golang/src/runtime/proc.go:290"}
{"level":"info","ts":"2026-11-03T03:32:13.186255267Z","logger":"setup","caller":"kueue/main.go:625","msg":"Probe endpoints are configured on healthz and readyz"}
{"level":"info","ts":"2026-11-03T03:32:13.186364025Z","logger":"queue-manager","caller":"queue/manager.go:138","msg":"DRA reconcile channel connected"}
{"level":"info","ts":"2026-11-03T03:32:13.235417621Z","logger":"setup","caller":"kueue/main.go:497","msg":"Skipping provisioning controller setup: Provisioning Requests not supported (Possible cause: missing or unsupported cluster-autoscaler)"}
{"level":"info","ts":"2026-11-03T03:32:13.235698063Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"workload.codeflare.dev/v1beta2, Kind=AppWrapper","path":"/mutate-workload-codeflare-dev-v1beta2-appwrapper"}
{"level":"info","ts":"2026-11-03T03:32:13.235833322Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-workload-codeflare-dev-v1beta2-appwrapper"}
{"level":"info","ts":"2026-11-03T03:32:13.235878203Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"workload.codeflare.dev/v1beta2, Kind=AppWrapper","path":"/validate-workload-codeflare-dev-v1beta2-appwrapper"}
{"level":"info","ts":"2026-11-03T03:32:13.235934725Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-workload-codeflare-dev-v1beta2-appwrapper"}
{"level":"info","ts":"2026-11-03T03:32:13.236080907Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"apps/v1, Kind=Deployment","path":"/mutate-apps-v1-deployment"}
{"level":"info","ts":"2026-11-03T03:32:13.236137176Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-apps-v1-deployment"}
{"level":"info","ts":"2026-11-03T03:32:13.236168445Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"apps/v1, Kind=Deployment","path":"/validate-apps-v1-deployment"}
{"level":"info","ts":"2026-11-03T03:32:13.236217911Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-apps-v1-deployment"}
{"level":"info","ts":"2026-11-03T03:32:13.236314372Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"batch/v1, Kind=Job","path":"/mutate-batch-v1-job"}
{"level":"info","ts":"2026-11-03T03:32:13.236373681Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-batch-v1-job"}
{"level":"info","ts":"2026-11-03T03:32:13.236406283Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"batch/v1, Kind=Job","path":"/validate-batch-v1-job"}
{"level":"info","ts":"2026-11-03T03:32:13.236451741Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-batch-v1-job"}
{"level":"info","ts":"2026-11-03T03:32:13.236546716Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"jobset.x-k8s.io/v1alpha2, Kind=JobSet","path":"/mutate-jobset-x-k8s-io-v1alpha2-jobset"}
{"level":"info","ts":"2026-11-03T03:32:13.236603215Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-jobset-x-k8s-io-v1alpha2-jobset"}
{"level":"info","ts":"2026-11-03T03:32:13.236637215Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"jobset.x-k8s.io/v1alpha2, Kind=JobSet","path":"/validate-jobset-x-k8s-io-v1alpha2-jobset"}
{"level":"info","ts":"2026-11-03T03:32:13.236694248Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-jobset-x-k8s-io-v1alpha2-jobset"}
{"level":"info","ts":"2026-11-03T03:32:13.23678456Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"kubeflow.org/v1, Kind=JAXJob","path":"/mutate-kubeflow-org-v1-jaxjob"}
{"level":"info","ts":"2026-11-03T03:32:13.236833452Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-kubeflow-org-v1-jaxjob"}
{"level":"info","ts":"2026-11-03T03:32:13.236855964Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"kubeflow.org/v1, Kind=JAXJob","path":"/validate-kubeflow-org-v1-jaxjob"}
{"level":"info","ts":"2026-11-03T03:32:13.23689256Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-kubeflow-org-v1-jaxjob"}
{"level":"info","ts":"2026-11-03T03:32:13.236943897Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"kubeflow.org/v1, Kind=PaddleJob","path":"/mutate-kubeflow-org-v1-paddlejob"}
{"level":"info","ts":"2026-11-03T03:32:13.23698053Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-kubeflow-org-v1-paddlejob"}
{"level":"info","ts":"2026-11-03T03:32:13.237002194Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"kubeflow.org/v1, Kind=PaddleJob","path":"/validate-kubeflow-org-v1-paddlejob"}
{"level":"info","ts":"2026-11-03T03:32:13.237039393Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-kubeflow-org-v1-paddlejob"}
{"level":"info","ts":"2026-11-03T03:32:13.237087478Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"kubeflow.org/v1, Kind=PyTorchJob","path":"/mutate-kubeflow-org-v1-pytorchjob"}
{"level":"info","ts":"2026-11-03T03:32:13.237126759Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-kubeflow-org-v1-pytorchjob"}
{"level":"info","ts":"2026-11-03T03:32:13.237149178Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"kubeflow.org/v1, Kind=PyTorchJob","path":"/validate-kubeflow-org-v1-pytorchjob"}
{"level":"info","ts":"2026-11-03T03:32:13.237201206Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-kubeflow-org-v1-pytorchjob"}
{"level":"info","ts":"2026-11-03T03:32:13.237270785Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"kubeflow.org/v1, Kind=TFJob","path":"/mutate-kubeflow-org-v1-tfjob"}
{"level":"info","ts":"2026-11-03T03:32:13.23734601Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-kubeflow-org-v1-tfjob"}
{"level":"info","ts":"2026-11-03T03:32:13.237375206Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"kubeflow.org/v1, Kind=TFJob","path":"/validate-kubeflow-org-v1-tfjob"}
{"level":"info","ts":"2026-11-03T03:32:13.237416096Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-kubeflow-org-v1-tfjob"}
{"level":"info","ts":"2026-11-03T03:32:13.237494123Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"kubeflow.org/v1, Kind=XGBoostJob","path":"/mutate-kubeflow-org-v1-xgboostjob"}
{"level":"info","ts":"2026-11-03T03:32:13.237546758Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-kubeflow-org-v1-xgboostjob"}
{"level":"info","ts":"2026-11-03T03:32:13.237572814Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"kubeflow.org/v1, Kind=XGBoostJob","path":"/validate-kubeflow-org-v1-xgboostjob"}
{"level":"info","ts":"2026-11-03T03:32:13.237607447Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-kubeflow-org-v1-xgboostjob"}
{"level":"info","ts":"2026-11-03T03:32:13.237657604Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"kubeflow.org/v2beta1, Kind=MPIJob","path":"/mutate-kubeflow-org-v2beta1-mpijob"}
{"level":"info","ts":"2026-11-03T03:32:13.237693961Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-kubeflow-org-v2beta1-mpijob"}
{"level":"info","ts":"2026-11-03T03:32:13.237730075Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"kubeflow.org/v2beta1, Kind=MPIJob","path":"/validate-kubeflow-org-v2beta1-mpijob"}
{"level":"info","ts":"2026-11-03T03:32:13.237779195Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-kubeflow-org-v2beta1-mpijob"}
{"level":"info","ts":"2026-11-03T03:32:13.237899578Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"/v1, Kind=Pod","path":"/mutate--v1-pod"}
{"level":"info","ts":"2026-11-03T03:32:13.237944982Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate--v1-pod"}
{"level":"info","ts":"2026-11-03T03:32:13.237967683Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"/v1, Kind=Pod","path":"/validate--v1-pod"}
{"level":"info","ts":"2026-11-03T03:32:13.238008578Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate--v1-pod"}
{"level":"info","ts":"2026-11-03T03:32:13.238057652Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"leaderworkerset.x-k8s.io/v1, Kind=LeaderWorkerSet","path":"/mutate-leaderworkerset-x-k8s-io-v1-leaderworkerset"}
{"level":"info","ts":"2026-11-03T03:32:13.238094362Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-leaderworkerset-x-k8s-io-v1-leaderworkerset"}
{"level":"info","ts":"2026-11-03T03:32:13.238115429Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"leaderworkerset.x-k8s.io/v1, Kind=LeaderWorkerSet","path":"/validate-leaderworkerset-x-k8s-io-v1-leaderworkerset"}
{"level":"info","ts":"2026-11-03T03:32:13.238148423Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-leaderworkerset-x-k8s-io-v1-leaderworkerset"}
{"level":"info","ts":"2026-11-03T03:32:13.238194574Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"ray.io/v1, Kind=RayCluster","path":"/mutate-ray-io-v1-raycluster"}
{"level":"info","ts":"2026-11-03T03:32:13.238231833Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-ray-io-v1-raycluster"}
{"level":"info","ts":"2026-11-03T03:32:13.238253486Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"ray.io/v1, Kind=RayCluster","path":"/validate-ray-io-v1-raycluster"}
{"level":"info","ts":"2026-11-03T03:32:13.238288962Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-ray-io-v1-raycluster"}
{"level":"info","ts":"2026-11-03T03:32:13.238336909Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"ray.io/v1, Kind=RayJob","path":"/mutate-ray-io-v1-rayjob"}
{"level":"info","ts":"2026-11-03T03:32:13.238373128Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-ray-io-v1-rayjob"}
{"level":"info","ts":"2026-11-03T03:32:13.23839464Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"ray.io/v1, Kind=RayJob","path":"/validate-ray-io-v1-rayjob"}
{"level":"info","ts":"2026-11-03T03:32:13.238430294Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-ray-io-v1-rayjob"}
{"level":"info","ts":"2026-11-03T03:32:13.238476007Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"ray.io/v1, Kind=RayService","path":"/mutate-ray-io-v1-rayservice"}
{"level":"info","ts":"2026-11-03T03:32:13.238525371Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-ray-io-v1-rayservice"}
{"level":"info","ts":"2026-11-03T03:32:13.238546156Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"ray.io/v1, Kind=RayService","path":"/validate-ray-io-v1-rayservice"}
{"level":"info","ts":"2026-11-03T03:32:13.238582569Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-ray-io-v1-rayservice"}
{"level":"info","ts":"2026-11-03T03:32:13.238630325Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"sparkoperator.k8s.io/v1beta2, Kind=SparkApplication","path":"/mutate-sparkoperator-k8s-io-v1beta2-sparkapplication"}
{"level":"info","ts":"2026-11-03T03:32:13.23866509Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-sparkoperator-k8s-io-v1beta2-sparkapplication"}
{"level":"info","ts":"2026-11-03T03:32:13.238686218Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"sparkoperator.k8s.io/v1beta2, Kind=SparkApplication","path":"/validate-sparkoperator-k8s-io-v1beta2-sparkapplication"}
{"level":"info","ts":"2026-11-03T03:32:13.238738971Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-sparkoperator-k8s-io-v1beta2-sparkapplication"}
{"level":"info","ts":"2026-11-03T03:32:13.238795348Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"apps/v1, Kind=StatefulSet","path":"/mutate-apps-v1-statefulset"}
{"level":"info","ts":"2026-11-03T03:32:13.238831752Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-apps-v1-statefulset"}
{"level":"info","ts":"2026-11-03T03:32:13.23885449Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"apps/v1, Kind=StatefulSet","path":"/validate-apps-v1-statefulset"}
{"level":"info","ts":"2026-11-03T03:32:13.238890826Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-apps-v1-statefulset"}
{"level":"info","ts":"2026-11-03T03:32:13.239229844Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","path":"/mutate-trainer-kubeflow-org-v1alpha1-trainjob"}
{"level":"info","ts":"2026-11-03T03:32:13.239280681Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-trainer-kubeflow-org-v1alpha1-trainjob"}
{"level":"info","ts":"2026-11-03T03:32:13.239304887Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","path":"/validate-trainer-kubeflow-org-v1alpha1-trainjob"}
{"level":"info","ts":"2026-11-03T03:32:13.239342248Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-trainer-kubeflow-org-v1alpha1-trainjob"}
{"level":"info","ts":"2026-11-03T03:32:13.239395997Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"kueue.x-k8s.io/v1beta2, Kind=Workload","path":"/mutate-kueue-x-k8s-io-v1beta2-workload"}
{"level":"info","ts":"2026-11-03T03:32:13.23943305Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-kueue-x-k8s-io-v1beta2-workload"}
{"level":"info","ts":"2026-11-03T03:32:13.23945521Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"kueue.x-k8s.io/v1beta2, Kind=Workload","path":"/validate-kueue-x-k8s-io-v1beta2-workload"}
{"level":"info","ts":"2026-11-03T03:32:13.239491822Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-kueue-x-k8s-io-v1beta2-workload"}
{"level":"info","ts":"2026-11-03T03:32:13.239562938Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/convert"}
{"level":"info","ts":"2026-11-03T03:32:13.239575687Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:319","msg":"Conversion webhook enabled","GVK":"kueue.x-k8s.io/v1beta2, Kind=Workload"}
{"level":"info","ts":"2026-11-03T03:32:13.239602837Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"kueue.x-k8s.io/v1beta2, Kind=ResourceFlavor","path":"/mutate-kueue-x-k8s-io-v1beta2-resourceflavor"}
{"level":"info","ts":"2026-11-03T03:32:13.239643682Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-kueue-x-k8s-io-v1beta2-resourceflavor"}
{"level":"info","ts":"2026-11-03T03:32:13.239671855Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"kueue.x-k8s.io/v1beta2, Kind=ResourceFlavor","path":"/validate-kueue-x-k8s-io-v1beta2-resourceflavor"}
{"level":"info","ts":"2026-11-03T03:32:13.239705723Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-kueue-x-k8s-io-v1beta2-resourceflavor"}
{"level":"info","ts":"2026-11-03T03:32:13.239771871Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"kueue.x-k8s.io/v1beta2, Kind=ClusterQueue","path":"/mutate-kueue-x-k8s-io-v1beta2-clusterqueue"}
{"level":"info","ts":"2026-11-03T03:32:13.239806193Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-kueue-x-k8s-io-v1beta2-clusterqueue"}
{"level":"info","ts":"2026-11-03T03:32:13.23983341Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"kueue.x-k8s.io/v1beta2, Kind=ClusterQueue","path":"/validate-kueue-x-k8s-io-v1beta2-clusterqueue"}
{"level":"info","ts":"2026-11-03T03:32:13.239875965Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-kueue-x-k8s-io-v1beta2-clusterqueue"}
{"level":"info","ts":"2026-11-03T03:32:13.239910692Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:319","msg":"Conversion webhook enabled","GVK":"kueue.x-k8s.io/v1beta2, Kind=ClusterQueue"}
{"level":"info","ts":"2026-11-03T03:32:13.239932564Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"kueue.x-k8s.io/v1beta2, Kind=Cohort","path":"/validate-kueue-x-k8s-io-v1beta2-cohort"}
{"level":"info","ts":"2026-11-03T03:32:13.239974705Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-kueue-x-k8s-io-v1beta2-cohort"}
{"level":"info","ts":"2026-11-03T03:32:13.240010647Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:319","msg":"Conversion webhook enabled","GVK":"kueue.x-k8s.io/v1beta2, Kind=Cohort"}
{"level":"info","ts":"2026-11-03T03:32:13.240040218Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:319","msg":"Conversion webhook enabled","GVK":"kueue.x-k8s.io/v1beta2, Kind=LocalQueue"}
{"level":"info","ts":"2026-11-03T03:32:13.240090634Z","logger":"setup","caller":"kueue/main.go:430","msg":"Starting manager"}
{"level":"info","ts":"2026-11-03T03:32:13.240168215Z","logger":"controller-runtime.metrics","caller":"server/server.go:208","msg":"Starting metrics server"}
{"level":"info","ts":"2026-11-03T03:32:13.240245069Z","caller":"manager/server.go:83","msg":"starting server","name":"health probe","addr":"[::]:8081"}
{"level":"info","ts":"2026-11-03T03:32:13.240322902Z","logger":"controller-runtime.metrics","caller":"server/server.go:247","msg":"Serving metrics server","bindAddress":":8443","secure":true}
{"level":"info","ts":"2026-11-03T03:32:13.240340525Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:191","msg":"Starting webhook server"}
{"level":"info","ts":"2026-11-03T03:32:13.240860811Z","logger":"controller-runtime.certwatcher","caller":"certwatcher/certwatcher.go:214","msg":"Updated current TLS certificate","cert":"/tmp/k8s-webhook-server/serving-certs/tls.crt","key":"/tmp/k8s-webhook-server/serving-certs/tls.key"}
{"level":"info","ts":"2026-11-03T03:32:13.24111969Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:242","msg":"Serving webhook server","host":"","port":9443}
{"level":"info","ts":"2026-11-03T03:32:13.241259279Z","logger":"controller-runtime.certwatcher","caller":"certwatcher/certwatcher.go:136","msg":"Starting certificate poll+watcher","cert":"/tmp/k8s-webhook-server/serving-certs/tls.crt","key":"/tmp/k8s-webhook-server/serving-certs/tls.key","interval":10}
{"level":"Level(-2)","ts":"2026-11-03T03:32:13.289907567Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1beta2.MultiKueueConfig","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-03T03:32:13.290128108Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1beta2.LocalQueue","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-03T03:32:13.290339717Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1beta2.MultiKueueCluster","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-03T03:32:13.290403265Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1.LimitRange","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-03T03:32:13.29043691Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1.Node","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-03T03:32:13.291091273Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1beta2.AdmissionCheck","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-03T03:32:13.29118694Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1alpha1.TrainingRuntime","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-03T03:32:13.291590194Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1beta2.ResourceFlavor","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-03T03:32:13.292098113Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1beta2.Workload","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-03T03:32:13.293012397Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1alpha1.TrainJob","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-03T03:32:13.293558387Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1alpha1.ClusterTrainingRuntime","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-03T03:32:13.431772925Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1.Pod","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
I1103 03:32:13.434260       1 plugins.go:157] Loaded 1 mutating admission controller(s) successfully in the following order: NamespaceLifecycle.
I1103 03:32:13.437053       1 handler.go:304] Adding GroupVersion visibility.kueue.x-k8s.io v1beta2 to ResourceManager
I1103 03:32:13.437303       1 handler.go:304] Adding GroupVersion visibility.kueue.x-k8s.io v1beta1 to ResourceManager
{"level":"info","ts":"2026-11-03T03:32:13.441290552Z","logger":"controller-runtime.certwatcher","caller":"certwatcher/certwatcher.go:136","msg":"Starting certificate poll+watcher","cert":"/etc/kueue/metrics/certs/tls.crt","key":"/etc/kueue/metrics/certs/tls.key","interval":10}
{"level":"info","ts":"2026-11-03T03:32:13.441430532Z","logger":"tas-node-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1.Pod"}
{"level":"info","ts":"2026-11-03T03:32:13.441509473Z","logger":"ElasticJobUngater","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1.Pod"}
{"level":"info","ts":"2026-11-03T03:32:13.44155941Z","logger":"tas-node-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1.Node"}
{"level":"info","ts":"2026-11-03T03:32:13.441611119Z","logger":"clusterqueue-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1beta2.ClusterQueue"}
{"level":"info","ts":"2026-11-03T03:32:13.44165749Z","logger":"workload-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1beta2.LocalQueue"}
{"level":"info","ts":"2026-11-03T03:32:13.441517735Z","logger":"tas-non-tas-usage-controller","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1.Pod"}
{"level":"info","ts":"2026-11-03T03:32:13.441426286Z","logger":"admissioncheck-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"channel source: 0x2ba257492540"}
{"level":"info","ts":"2026-11-03T03:32:13.441546887Z","logger":"clusterqueue-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"channel source: 0x2ba257492a10"}
{"level":"info","ts":"2026-11-03T03:32:13.441570282Z","logger":"cohort-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"channel source: 0x2ba257492930"}
{"level":"info","ts":"2026-11-03T03:32:13.441815578Z","logger":"tas-resource-flavor-controller","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"channel source: 0x2ba257493260"}
{"level":"info","ts":"2026-11-03T03:32:13.441441088Z","logger":"admissioncheck-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1beta2.AdmissionCheck"}
{"level":"info","ts":"2026-11-03T03:32:13.441588607Z","logger":"clusterqueue-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1.Namespace"}
{"level":"info","ts":"2026-11-03T03:32:13.441852667Z","logger":"tas-resource-flavor-controller","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1beta2.ResourceFlavor"}
{"level":"info","ts":"2026-11-03T03:32:13.441598399Z","logger":"workload-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"channel source: 0x2ba257492bd0"}
{"level":"info","ts":"2026-11-03T03:32:13.441609999Z","logger":"cohort-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1beta2.Cohort"}
{"level":"info","ts":"2026-11-03T03:32:13.441584413Z","logger":"localqueue-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"channel source: 0x2ba2574927e0"}
{"level":"info","ts":"2026-11-03T03:32:13.441600478Z","logger":"resourceflavor-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"channel source: 0x2ba2574923f0"}
{"level":"info","ts":"2026-11-03T03:32:13.44163678Z","logger":"workload-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1.LimitRange"}
{"level":"info","ts":"2026-11-03T03:32:13.441641001Z","logger":"workload-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1beta2.ClusterQueue"}
I1103 03:32:13.441453       1 leaderelection.go:258] "Attempting to acquire leader lease..." lock="openshift-kueue-operator/c1f6bfd2.kueue.x-k8s.io"
{"level":"info","ts":"2026-11-03T03:32:13.441661875Z","logger":"workloadpriorityclass-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1beta2.WorkloadPriorityClass"}
{"level":"info","ts":"2026-11-03T03:32:13.441466754Z","logger":"ElasticJobUngater","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1beta2.Workload"}
{"level":"info","ts":"2026-11-03T03:32:13.441680859Z","logger":"tas-topology-controller","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1beta2.Topology"}
{"level":"info","ts":"2026-11-03T03:32:13.441691722Z","logger":"localqueue-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1beta2.ClusterQueue"}
{"level":"info","ts":"2026-11-03T03:32:13.441938579Z","logger":"tas-topology-ungater","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1beta2.Workload"}
{"level":"info","ts":"2026-11-03T03:32:13.441704541Z","logger":"localqueue-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1beta2.LocalQueue"}
{"level":"info","ts":"2026-11-03T03:32:13.441740746Z","logger":"tas-topology-controller","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1beta2.ResourceFlavor"}
{"level":"info","ts":"2026-11-03T03:32:13.441958897Z","logger":"tas-topology-ungater","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1.Pod"}
{"level":"info","ts":"2026-11-03T03:32:13.441751052Z","logger":"workload-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1beta2.Workload"}
{"level":"info","ts":"2026-11-03T03:32:13.441626589Z","logger":"resourceflavor-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1beta2.ResourceFlavor"}
{"level":"info","ts":"2026-11-03T03:32:13.441666366Z","logger":"workload-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1.RuntimeClass"}
I1103 03:32:13.488908       1 requestheader_controller.go:180] Starting RequestHeaderAuthRequestController
I1103 03:32:13.488921       1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file"
I1103 03:32:13.488938       1 shared_informer.go:402] "Waiting for caches to sync"
I1103 03:32:13.488925       1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::client-ca-file"
I1103 03:32:13.488973       1 shared_informer.go:402] "Waiting for caches to sync"
I1103 03:32:13.488941       1 shared_informer.go:402] "Waiting for caches to sync"
I1103 03:32:13.489267       1 dynamic_serving_content.go:135] "Starting controller" name="serving-cert::/visibility/tls.crt::/visibility/tls.key"
I1103 03:32:13.490111       1 secure_serving.go:214] Serving securely on [::]:8082
I1103 03:32:13.490153       1 tlsconfig.go:243] "Starting DynamicServingCertificateController"
I1103 03:32:13.490217       1 apf_controller.go:377] Starting API Priority and Fairness config controller
{"level":"Level(-2)","ts":"2026-11-03T03:32:13.490828518Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1beta2.Cohort","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-03T03:32:13.490921988Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1.RuntimeClass","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-03T03:32:13.491002558Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1beta2.ClusterQueue","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-03T03:32:13.491071127Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1beta2.WorkloadPriorityClass","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-03T03:32:13.491176357Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1beta2.Topology","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-03T03:32:13.492847641Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1.Namespace","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"info","ts":"2026-11-03T03:32:13.542140755Z","logger":"admissioncheck-reconciler","caller":"controller/controller.go:302","msg":"Starting Controller","replica-role":"follower"}
{"level":"info","ts":"2026-11-03T03:32:13.542170892Z","logger":"admissioncheck-reconciler","caller":"controller/controller.go:305","msg":"Starting workers","replica-role":"follower","worker count":1}
{"level":"info","ts":"2026-11-03T03:32:13.542125327Z","logger":"tas-node-reconciler","caller":"controller/controller.go:302","msg":"Starting Controller","replica-role":"follower"}
{"level":"info","ts":"2026-11-03T03:32:13.542191717Z","logger":"tas-node-reconciler","caller":"controller/controller.go:305","msg":"Starting workers","replica-role":"follower","worker count":1}
{"level":"info","ts":"2026-11-03T03:32:13.54217138Z","logger":"tas-resource-flavor-controller","caller":"controller/controller.go:302","msg":"Starting Controller","replica-role":"follower"}
{"level":"info","ts":"2026-11-03T03:32:13.542191299Z","logger":"tas-non-tas-usage-controller","caller":"controller/controller.go:302","msg":"Starting Controller","replica-role":"follower"}
{"level":"info","ts":"2026-11-03T03:32:13.542192666Z","logger":"resourceflavor-reconciler","caller":"controller/controller.go:302","msg":"Starting Controller","replica-role":"follower"}
{"level":"info","ts":"2026-11-03T03:32:13.542217385Z","logger":"resourceflavor-reconciler","caller":"controller/controller.go:305","msg":"Starting workers","replica-role":"follower","worker count":1}
{"level":"info","ts":"2026-11-03T03:32:13.542203526Z","logger":"tas-resource-flavor-controller","caller":"controller/controller.go:305","msg":"Starting workers","replica-role":"follower","worker count":1}
{"level":"info","ts":"2026-11-03T03:32:13.542206229Z","logger":"tas-non-tas-usage-controller","caller":"controller/controller.go:305","msg":"Starting workers","replica-role":"follower","worker count":5}
{"level":"info","ts":"2026-11-03T03:32:13.542423598Z","logger":"tas-topology-ungater","caller":"controller/controller.go:302","msg":"Starting Controller","replica-role":"follower"}
{"level":"info","ts":"2026-11-03T03:32:13.542447202Z","logger":"tas-topology-ungater","caller":"controller/controller.go:305","msg":"Starting workers","replica-role":"follower","worker count":5}
{"level":"info","ts":"2026-11-03T03:32:13.542467609Z","logger":"ElasticJobUngater","caller":"controller/controller.go:302","msg":"Starting Controller","replica-role":"follower"}
{"level":"info","ts":"2026-11-03T03:32:13.542505082Z","logger":"ElasticJobUngater","caller":"controller/controller.go:305","msg":"Starting workers","replica-role":"follower","worker count":5}
I1103 03:32:13.589084       1 shared_informer.go:409] "Caches are synced"
I1103 03:32:13.589093       1 shared_informer.go:409] "Caches are synced"
I1103 03:32:13.589095       1 shared_informer.go:409] "Caches are synced"
I1103 03:32:13.590478       1 apf_controller.go:382] Running API Priority and Fairness config worker
I1103 03:32:13.590491       1 apf_controller.go:385] Running API Priority and Fairness periodic rebalancing process
{"level":"info","ts":"2026-11-03T03:32:13.643108531Z","logger":"clusterqueue-reconciler","caller":"controller/controller.go:302","msg":"Starting Controller","replica-role":"follower"}
{"level":"info","ts":"2026-11-03T03:32:13.643197237Z","logger":"clusterqueue-reconciler","caller":"controller/controller.go:305","msg":"Starting workers","replica-role":"follower","worker count":1}
{"level":"info","ts":"2026-11-03T03:32:13.643099954Z","logger":"localqueue-reconciler","caller":"controller/controller.go:302","msg":"Starting Controller","replica-role":"follower"}
{"level":"info","ts":"2026-11-03T03:32:13.643108556Z","logger":"cohort-reconciler","caller":"controller/controller.go:302","msg":"Starting Controller","replica-role":"follower"}
{"level":"info","ts":"2026-11-03T03:32:13.643224528Z","logger":"localqueue-reconciler","caller":"controller/controller.go:305","msg":"Starting workers","replica-role":"follower","worker count":1}
{"level":"info","ts":"2026-11-03T03:32:13.643140329Z","logger":"workload-reconciler","caller":"controller/controller.go:302","msg":"Starting Controller","replica-role":"follower"}
{"level":"info","ts":"2026-11-03T03:32:13.643234604Z","logger":"cohort-reconciler","caller":"controller/controller.go:305","msg":"Starting workers","replica-role":"follower","worker count":1}
{"level":"info","ts":"2026-11-03T03:32:13.643242879Z","logger":"workload-reconciler","caller":"controller/controller.go:305","msg":"Starting workers","replica-role":"follower","worker count":5}
{"level":"info","ts":"2026-11-03T03:32:13.643109965Z","logger":"tas-topology-controller","caller":"controller/controller.go:302","msg":"Starting Controller","replica-role":"follower"}
{"level":"info","ts":"2026-11-03T03:32:13.643159008Z","logger":"workloadpriorityclass-reconciler","caller":"controller/controller.go:302","msg":"Starting Controller","replica-role":"follower"}
{"level":"info","ts":"2026-11-03T03:32:13.643254775Z","logger":"tas-topology-controller","caller":"controller/controller.go:305","msg":"Starting workers","replica-role":"follower","worker count":1}
{"level":"info","ts":"2026-11-03T03:32:13.643260462Z","logger":"workloadpriorityclass-reconciler","caller":"controller/controller.go:305","msg":"Starting workers","replica-role":"follower","worker count":1}
{"level":"Level(-2)","ts":"2026-11-03T04:07:52.54632238Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:141","msg":"ResourceFlavor create event","replica-role":"follower","resourceFlavor":{"name":"rf-fkql6"}}
{"level":"Level(-2)","ts":"2026-11-03T04:07:52.894307333Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:351","msg":"ClusterQueue create event","replica-role":"follower","clusterQueue":{"name":"cq-m2r6r"}}
{"level":"Level(-2)","ts":"2026-11-03T04:07:52.912136072Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:240","msg":"LocalQueue create event","replica-role":"follower","localQueue":{"name":"lq-gntg5","namespace":"test-ns-pj9cs"}}
{"level":"Level(-2)","ts":"2026-11-03T04:07:52.953953518Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"follower","clusterQueue":{"name":"cq-m2r6r"}}
{"level":"Level(-2)","ts":"2026-11-03T04:07:52.968605384Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"follower","localQueue":{"name":"lq-gntg5","namespace":"test-ns-pj9cs"}}
{"level":"Level(-2)","ts":"2026-11-03T04:07:53.027150559Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"follower","localQueue":{"name":"lq-gntg5","namespace":"test-ns-pj9cs"}}
{"level":"Level(-2)","ts":"2026-11-03T04:07:54.285570466Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1276","msg":"Workload create event","replica-role":"follower","workload":{"name":"pod-dw-deployment-rnsbp-d7d46fb8-777pw-383b1","namespace":"test-ns-pj9cs"},"queue":"lq-gntg5","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-03T04:07:54.457623997Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"follower","workload":{"name":"pod-dw-deployment-rnsbp-d7d46fb8-777pw-383b1","namespace":"test-ns-pj9cs"},"queue":"lq-gntg5","status":"admitted","prevStatus":"pending","clusterQueue":"cq-m2r6r"}
{"level":"Level(-2)","ts":"2026-11-03T04:07:55.394210308Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"follower","localQueue":{"name":"lq-gntg5","namespace":"test-ns-pj9cs"}}
{"level":"Level(-2)","ts":"2026-11-03T04:07:55.398495093Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"follower","clusterQueue":{"name":"cq-m2r6r"}}
{"level":"Level(-2)","ts":"2026-11-03T04:09:16.382588003Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1276","msg":"Workload create event","replica-role":"follower","workload":{"name":"trainjob-v21bdac94b06-a0214","namespace":"test-ns-pj9cs"},"queue":"lq-gntg5","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:16.504921749Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"follower","workload":{"name":"trainjob-v21bdac94b06-a0214","namespace":"test-ns-pj9cs"},"queue":"lq-gntg5","status":"admitted","prevStatus":"pending","clusterQueue":"cq-m2r6r"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:17.439567232Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"follower","localQueue":{"name":"lq-gntg5","namespace":"test-ns-pj9cs"}}
{"level":"Level(-2)","ts":"2026-11-03T04:09:17.485305554Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"follower","clusterQueue":{"name":"cq-m2r6r"}}
{"level":"Level(-2)","ts":"2026-11-03T04:09:47.901956433Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"follower","workload":{"name":"trainjob-v21bdac94b06-a0214","namespace":"test-ns-pj9cs"},"queue":"lq-gntg5","status":"finished","prevStatus":"admitted","clusterQueue":"cq-m2r6r"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:48.180524608Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"follower","workload":{"name":"trainjob-v21bdac94b06-a0214","namespace":"test-ns-pj9cs"},"queue":"lq-gntg5","status":"finished","clusterQueue":"cq-m2r6r"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:48.960134859Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"follower","localQueue":{"name":"lq-gntg5","namespace":"test-ns-pj9cs"}}
{"level":"Level(-2)","ts":"2026-11-03T04:09:48.962914394Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"follower","clusterQueue":{"name":"cq-m2r6r"}}
{"level":"Level(-2)","ts":"2026-11-03T04:09:49.522496825Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1312","msg":"Workload delete event","replica-role":"follower","workload":{"name":"trainjob-v21bdac94b06-a0214","namespace":"test-ns-pj9cs"},"queue":"lq-gntg5","status":"finished"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:50.499564145Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"follower","clusterQueue":{"name":"cq-m2r6r"}}
{"level":"Level(-2)","ts":"2026-11-03T04:09:50.520555304Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:171","msg":"ResourceFlavor update event","replica-role":"follower","resourceFlavor":{"name":"rf-fkql6"}}
{"level":"Level(-2)","ts":"2026-11-03T04:09:50.559052262Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"follower","clusterQueue":{"name":"cq-m2r6r"}}
{"level":"Level(-2)","ts":"2026-11-03T04:09:50.617513809Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"follower","localQueue":{"name":"lq-gntg5","namespace":"test-ns-pj9cs"}}
{"level":"Level(-2)","ts":"2026-11-03T04:09:51.147108665Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:141","msg":"ResourceFlavor create event","replica-role":"follower","resourceFlavor":{"name":"rf-j9bhm"}}
{"level":"Level(-2)","ts":"2026-11-03T04:09:51.49589597Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:351","msg":"ClusterQueue create event","replica-role":"follower","clusterQueue":{"name":"cq-s4rfg"}}
{"level":"Level(-2)","ts":"2026-11-03T04:09:51.513236411Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:240","msg":"LocalQueue create event","replica-role":"follower","localQueue":{"name":"lq-7qwch","namespace":"test-ns-289zf"}}
{"level":"Level(-2)","ts":"2026-11-03T04:09:51.554793554Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"follower","clusterQueue":{"name":"cq-s4rfg"}}
{"level":"Level(-2)","ts":"2026-11-03T04:09:51.57122766Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"follower","localQueue":{"name":"lq-7qwch","namespace":"test-ns-289zf"}}
{"level":"Level(-2)","ts":"2026-11-03T04:09:51.627851067Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"follower","localQueue":{"name":"lq-7qwch","namespace":"test-ns-289zf"}}
{"level":"Level(-2)","ts":"2026-11-03T04:09:52.026182057Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1276","msg":"Workload create event","replica-role":"follower","workload":{"name":"trainjob-test-preemption-trainjob-fnt6p-dced9","namespace":"test-ns-289zf"},"queue":"lq-7qwch","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:52.138317981Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"follower","workload":{"name":"trainjob-test-preemption-trainjob-fnt6p-dced9","namespace":"test-ns-289zf"},"queue":"lq-7qwch","status":"admitted","prevStatus":"pending","clusterQueue":"cq-s4rfg"}
{"level":"Level(-2)","ts":"2026-11-03T04:09:53.085784968Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"follower","localQueue":{"name":"lq-7qwch","namespace":"test-ns-289zf"}}
{"level":"Level(-2)","ts":"2026-11-03T04:09:53.088258404Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"follower","clusterQueue":{"name":"cq-s4rfg"}}
{"level":"Level(-2)","ts":"2026-11-03T04:10:21.411218024Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"follower","workload":{"name":"pod-dw-deployment-rnsbp-d7d46fb8-777pw-383b1","namespace":"test-ns-pj9cs"},"queue":"lq-gntg5","status":"admitted","clusterQueue":"cq-m2r6r"}
{"level":"Level(-2)","ts":"2026-11-03T04:10:21.595215287Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1312","msg":"Workload delete event","replica-role":"follower","workload":{"name":"pod-dw-deployment-rnsbp-d7d46fb8-777pw-383b1","namespace":"test-ns-pj9cs"},"queue":"lq-gntg5","status":"admitted"}
{"level":"Level(-2)","ts":"2026-11-03T04:10:22.474366999Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"follower","localQueue":{"name":"lq-gntg5","namespace":"test-ns-pj9cs"}}
{"level":"Level(-2)","ts":"2026-11-03T04:10:22.527530966Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:376","msg":"ClusterQueue delete event","replica-role":"follower","clusterQueue":{"name":"cq-m2r6r"}}
{"level":"Level(-2)","ts":"2026-11-03T04:10:22.527628887Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:385","msg":"Cleared resource metrics for deleted ClusterQueue.","replica-role":"follower","clusterQueue":{"name":"cq-m2r6r"}}
{"level":"Level(-2)","ts":"2026-11-03T04:10:22.5831695Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"follower","localQueue":{"name":"lq-gntg5","namespace":"test-ns-pj9cs"}}
{"level":"Level(-2)","ts":"2026-11-03T04:10:22.631151529Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:159","msg":"ResourceFlavor delete event","replica-role":"follower","resourceFlavor":{"name":"rf-fkql6"}}
{"level":"Level(-2)","ts":"2026-11-03T04:10:33.642931717Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:273","msg":"LocalQueue delete event","replica-role":"follower","localQueue":{"name":"lq-gntg5","namespace":"test-ns-pj9cs"}}
{"level":"Level(-2)","ts":"2026-11-03T04:15:08.171707411Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"follower","workload":{"name":"trainjob-test-preemption-trainjob-fnt6p-dced9","namespace":"test-ns-289zf"},"queue":"lq-7qwch","status":"admitted","clusterQueue":"cq-s4rfg"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:08.171802023Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1381","msg":"Workload will not be queued because the workload is not active","replica-role":"follower","workload":{"name":"trainjob-test-preemption-trainjob-fnt6p-dced9","namespace":"test-ns-289zf"},"queue":"lq-7qwch","status":"admitted","clusterQueue":"cq-s4rfg"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:08.288974838Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"follower","workload":{"name":"trainjob-test-preemption-trainjob-fnt6p-dced9","namespace":"test-ns-289zf"},"queue":"lq-7qwch","status":"admitted","clusterQueue":"cq-s4rfg"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:08.289035035Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1381","msg":"Workload will not be queued because the workload is not active","replica-role":"follower","workload":{"name":"trainjob-test-preemption-trainjob-fnt6p-dced9","namespace":"test-ns-289zf"},"queue":"lq-7qwch","status":"admitted","clusterQueue":"cq-s4rfg"}
{"level":"info","ts":"2026-11-03T04:15:08.950214983Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-preemption-trainjob-fnt6p","namespace":"test-ns-289zf"},"namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"d016b59d-d184-4b84-94ff-db108799595a","replica-role":"follower"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:09.231501346Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"follower","localQueue":{"name":"lq-7qwch","namespace":"test-ns-289zf"}}
{"level":"Level(-2)","ts":"2026-11-03T04:15:09.234222308Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"follower","clusterQueue":{"name":"cq-s4rfg"}}
{"level":"info","ts":"2026-11-03T04:15:10.249051049Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-preemption-trainjob-fnt6p","namespace":"test-ns-289zf"},"namespace":"test-ns-289zf","name":"test-preemption-trainjob-fnt6p","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"bf1d5f9a-bb5e-4799-8f0a-908f47f3e4ad","replica-role":"follower"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.254189927Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"follower","clusterQueue":{"name":"cq-s4rfg"}}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.270622364Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:171","msg":"ResourceFlavor update event","replica-role":"follower","resourceFlavor":{"name":"rf-j9bhm"}}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.416563024Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"follower","workload":{"name":"trainjob-test-preemption-trainjob-fnt6p-dced9","namespace":"test-ns-289zf"},"queue":"lq-7qwch","status":"pending","prevStatus":"admitted","prevClusterQueue":"cq-s4rfg"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.41662564Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1381","msg":"Workload will not be queued because the workload is not active","replica-role":"follower","workload":{"name":"trainjob-test-preemption-trainjob-fnt6p-dced9","namespace":"test-ns-289zf"},"queue":"lq-7qwch","status":"pending","prevStatus":"admitted","prevClusterQueue":"cq-s4rfg"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.48374613Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:376","msg":"ClusterQueue delete event","replica-role":"follower","clusterQueue":{"name":"cq-s4rfg"}}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.48382691Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:385","msg":"Cleared resource metrics for deleted ClusterQueue.","replica-role":"follower","clusterQueue":{"name":"cq-s4rfg"}}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.531616603Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"follower","workload":{"name":"trainjob-test-preemption-trainjob-fnt6p-dced9","namespace":"test-ns-289zf"},"queue":"lq-7qwch","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.531673866Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1381","msg":"Workload will not be queued because the workload is not active","replica-role":"follower","workload":{"name":"trainjob-test-preemption-trainjob-fnt6p-dced9","namespace":"test-ns-289zf"},"queue":"lq-7qwch","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.541342515Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"follower","localQueue":{"name":"lq-7qwch","namespace":"test-ns-289zf"}}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.646078336Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"follower","workload":{"name":"trainjob-test-preemption-trainjob-fnt6p-dced9","namespace":"test-ns-289zf"},"queue":"lq-7qwch","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.646142683Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1381","msg":"Workload will not be queued because the workload is not active","replica-role":"follower","workload":{"name":"trainjob-test-preemption-trainjob-fnt6p-dced9","namespace":"test-ns-289zf"},"queue":"lq-7qwch","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.697198586Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:159","msg":"ResourceFlavor delete event","replica-role":"follower","resourceFlavor":{"name":"rf-j9bhm"}}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.744656547Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:141","msg":"ResourceFlavor create event","replica-role":"follower","resourceFlavor":{"name":"rf-58n95"}}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.97320619Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:351","msg":"ClusterQueue create event","replica-role":"follower","clusterQueue":{"name":"cq-vkftm"}}
{"level":"Level(-2)","ts":"2026-11-03T04:15:10.990838988Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:240","msg":"LocalQueue create event","replica-role":"follower","localQueue":{"name":"lq-z6n2r","namespace":"test-ns-xglkq"}}
{"level":"Level(-2)","ts":"2026-11-03T04:15:11.032436399Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"follower","clusterQueue":{"name":"cq-vkftm"}}
{"level":"Level(-2)","ts":"2026-11-03T04:15:11.048730751Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"follower","localQueue":{"name":"lq-z6n2r","namespace":"test-ns-xglkq"}}
{"level":"Level(-2)","ts":"2026-11-03T04:15:11.104941293Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"follower","localQueue":{"name":"lq-z6n2r","namespace":"test-ns-xglkq"}}
{"level":"info","ts":"2026-11-03T04:15:11.232156182Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:117","msg":"Validating create","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-runtimepatches-trainjob-zx4cq","namespace":"test-ns-xglkq"},"namespace":"test-ns-xglkq","name":"test-runtimepatches-trainjob-zx4cq","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:admin","requestID":"6cd63d25-b0a6-4a4e-9ef5-76fa9ba37811","replica-role":"follower"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:11.602246434Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1276","msg":"Workload create event","replica-role":"follower","workload":{"name":"trainjob-test-runtimepatches-trainjob-zx4cq-b2e76","namespace":"test-ns-xglkq"},"queue":"lq-z6n2r","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:11.714580498Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"follower","workload":{"name":"trainjob-test-runtimepatches-trainjob-zx4cq-b2e76","namespace":"test-ns-xglkq"},"queue":"lq-z6n2r","status":"admitted","prevStatus":"pending","clusterQueue":"cq-vkftm"}
{"level":"info","ts":"2026-11-03T04:15:12.083329097Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-runtimepatches-trainjob-zx4cq","namespace":"test-ns-xglkq"},"namespace":"test-ns-xglkq","name":"test-runtimepatches-trainjob-zx4cq","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"bfe13b8b-c844-4a1f-923e-c8adb3852954","replica-role":"follower"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:12.363925095Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"follower","clusterQueue":{"name":"cq-vkftm"}}
{"level":"Level(-2)","ts":"2026-11-03T04:15:12.380554338Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:171","msg":"ResourceFlavor update event","replica-role":"follower","resourceFlavor":{"name":"rf-58n95"}}
{"level":"Level(-2)","ts":"2026-11-03T04:15:12.423690835Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"follower","clusterQueue":{"name":"cq-vkftm"}}
{"level":"info","ts":"2026-11-03T04:15:12.464118673Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-runtimepatches-trainjob-zx4cq","namespace":"test-ns-xglkq"},"namespace":"test-ns-xglkq","name":"test-runtimepatches-trainjob-zx4cq","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"9fbf6f42-4986-4b82-97ba-479394691411","replica-role":"follower"}
{"level":"Level(-2)","ts":"2026-11-03T04:15:12.484520909Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"follower","localQueue":{"name":"lq-z6n2r","namespace":"test-ns-xglkq"}}
{"level":"info","ts":"2026-11-03T04:15:12.627918136Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-runtimepatches-trainjob-zx4cq","namespace":"test-ns-xglkq"},"namespace":"test-ns-xglkq","name":"test-runtimepatches-trainjob-zx4cq","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"e9a946c7-9426-421b-957f-f5b1090de55f","replica-role":"follower"}