{"level":"info","ts":"2026-11-04T03:47:26.312094257Z","logger":"setup","caller":"kueue/main.go:721","msg":"Configuration loaded","config":"apiVersion: config.kueue.x-k8s.io/v1beta2\nclientConnection:\n  burst: 100\n  qps: 50\ncontroller:\n  groupKindConcurrency:\n    ClusterQueue.kueue.x-k8s.io: 1\n    Job.batch: 5\n    LocalQueue.kueue.x-k8s.io: 1\n    Pod: 5\n    ResourceFlavor.kueue.x-k8s.io: 1\n    Workload.kueue.x-k8s.io: 5\nhealth:\n  healthProbeBindAddress: :8081\nintegrations:\n  frameworks:\n  - trainer.kubeflow.org/trainjob\n  - deployment\ninternalCertManagement:\n  enable: false\nkind: Configuration\nleaderElection:\n  leaderElect: true\n  leaseDuration: 2m17s\n  renewDeadline: 1m47s\n  resourceLock: leases\n  resourceName: c1f6bfd2.kueue.x-k8s.io\n  resourceNamespace: \"\"\n  retryPeriod: 26s\nmanageJobsWithoutQueueName: false\nmanagedJobsNamespaceSelector:\n  matchLabels:\n    kueue.openshift.io/managed: \"true\"\nmetrics:\n  bindAddress: :8443\n  enableClusterQueueResources: true\n  localQueueMetrics:\n    enable: true\nmultiKueue:\n  dispatcherName: kueue.x-k8s.io/multikueue-dispatcher-all-at-once\n  gcInterval: 1m0s\n  origin: multikueue\n  workerLostTimeout: 15m0s\nnamespace: openshift-kueue-operator\ntls:\n  cipherSuites:\n  - TLS_AES_128_GCM_SHA256\n  - TLS_AES_256_GCM_SHA384\n  - TLS_CHACHA20_POLY1305_SHA256\n  - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256\n  - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256\n  - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384\n  - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\n  - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256\n  - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256\n  minVersion: VersionTLS12\nvisibilityServer:\n  bindPort: 8082\nwebhook:\n  certDir: /tmp/k8s-webhook-server/serving-certs\n  port: 9443\n"}
{"level":"info","ts":"2026-11-04T03:47:26.312316628Z","logger":"setup","caller":"kueue/main.go:170","msg":"Initializing","gitVersion":"d12900d9-dirty","gitCommit":"d12900d94cd6813a3882a6391ffd70009a4b61d9","buildDate":"2026-10-05T16:41:45Z"}
{"level":"Level(-2)","ts":"2026-11-04T03:47:26.312887846Z","logger":"setup","caller":"features/kube_features.go:1144","msg":"Loaded feature gates","featureGates":{"AdmissionFairSharing":true,"AdmissionGatedBy":false,"AssignQueueLabelsForPods":true,"CleanupProvisioningRequestsOnEviction":true,"ConcurrentAdmission":false,"CustomMetricLabels":false,"DRAExtendedResources":false,"DeferRayServiceFinalizationForRedisCleanup":true,"DeploymentJobUIDLabel":false,"DynamicResourceAllocation":false,"ElasticJobsViaWorkloadSlices":true,"ElasticJobsViaWorkloadSlicesWithTAS":false,"EnforceProvisioningPodTemplateContents":false,"FailureRecoveryPolicy":false,"FairSharingPreemptWithinNominal":true,"FairSharingPrioritizeNonBorrowing":true,"FairSharingReevaluatePreemptionCandidates":false,"FastQuotaReleaseInPodIntegration":false,"FinalizeTerminatingPodGroups":false,"FinishOrphanedWorkloads":true,"FlavorFungibility":true,"FlavorFungibilityPreserveScanProgress":true,"HierarchicalCohorts":true,"HighMaxParallelismWithinReconcile":false,"KueueDRAIntegration":true,"KueueDRAIntegrationExtendedResource":false,"KueueDRAIntegrationPartitionableDevices":false,"KueueDRARejectWorkloadsWhenDRADisabled":true,"LWSImmutableGroupSize":true,"LendingLimit":true,"LocalQueueDefaulting":true,"LocalQueueMetrics":true,"ManagedJobsNamespaceSelectorAlwaysRespected":true,"MetricForWorkloadCreationLatency":true,"MetricsForCohorts":true,"MultiKueue":true,"MultiKueueAdaptersForCustomJobs":true,"MultiKueueAllowInsecureKubeconfigs":false,"MultiKueueBatchJobClearingTTLSecondsAfterFinishedOnWorkerCluster":false,"MultiKueueBatchJobWithManagedBy":true,"MultiKueueClusterProfile":false,"MultiKueueManagerQuotaAutomation":false,"MultiKueueOrchestratedPreemption":false,"MultiKueueRedoAdmissionOnEvictionInWorker":true,"MultiKueueRemoteSpecSync":true,"MultiKueueReuseClientConnectionConfigForWorkers":false,"MultiKueueWaitForWorkloadAdmitted":true,"ObjectRetentionPolicies":true,"PartialAdmission":true,"PodGroupSchedulingShapeOrdering":false,"PodIntegrationCountSucceededPodsAsReady":false,"PodIntegrationValidateGroupOwner":true,"PrioritizePreemptorWorkloads":false,"PriorityBoost":false,"PrioritySortingWithinCohort":true,"PropagateBatchJobLabelsToWorkload":true,"QuotaCheckStrategy":false,"ReclaimablePods":true,"RecomputeAssignmentUponPreemptionTargetsOverlap":false,"RejectUpdatesToCQWithInvalidOnFlavors":false,"RemoveFinalizersWithStrictPatch":true,"ReservedResourceNameValidation":false,"SanitizePodSets":true,"SchedulerLongRequeueInterval":false,"SchedulerTimestampPreemptionBuffer":false,"SchedulingEquivalenceHashing":true,"SchedulingEquivalenceHashingIgnorePodSetName":false,"ShortWorkloadNames":false,"SkipAncestorCheckForDeletedWorkloads":true,"SkipChildJobSuspension":false,"SkipFinalizersForPodsSuspendedByParent":true,"SkipReassignmentForPodOwnedWorkloads":false,"SparkApplicationIntegration":false,"TASBalancedPlacement":false,"TASCacheNodeMatchResults":true,"TASCacheTopologyTree":false,"TASCachingRemainingResources":true,"TASFailedNodeReplacement":true,"TASFailedNodeReplacementFailFast":true,"TASGroupedPodSetSlicing":false,"TASHandleOverlappingFlavors":true,"TASMultiLayerTopology":false,"TASPartialSlices":true,"TASProfileMixed":true,"TASRecomputeAssignmentWithinSchedulingCycle":true,"TASRejectFalseUnconstrainedTopology":false,"TASReplaceNodeDueToNotReadyOverFixedTime":true,"TASReplaceNodeOnNodeTaints":true,"TASReplaceNodeOnPodTermination":true,"TASRespectNodeAffinityPreferred":false,"TASValidateWorkloadSliceSize":true,"TLSOptions":true,"TopologyAwareScheduling":true,"UnadmittedWorkloadsExplicitStatus":false,"UnadmittedWorkloadsObservability":false,"ValidateRayAndSparkJobUpdates":true,"VectorizedResourceRequests":true,"VisibilityOnDemand":true,"WorkloadIdentifierAnnotations":true,"WorkloadPriorityClassDefaulting":false,"WorkloadRequestUseMergePatch":false,"WorkloadValidateResourcesAreNonNegative":true,"WorkloadValidationForPodSetMetadata":true}}
{"level":"info","ts":"2026-11-04T03:47:26.313006187Z","logger":"setup","caller":"kueue/main.go:213","msg":"Initializing metrics certificate watcher using provided certificates","metrics-cert-path":"/etc/kueue/metrics/certs"}
{"level":"info","ts":"2026-11-04T03:47:26.313377328Z","logger":"controller-runtime.certwatcher","caller":"certwatcher/certwatcher.go:214","msg":"Updated current TLS certificate","cert":"/etc/kueue/metrics/certs/tls.crt","key":"/etc/kueue/metrics/certs/tls.key"}
{"level":"Level(-2)","ts":"2026-11-04T03:47:26.31374393Z","logger":"setup","caller":"kueue/main.go:257","msg":"K8S Client","qps":50,"burst":100}
{"level":"info","ts":"2026-11-04T03:47:26.654196093Z","logger":"setup","caller":"kueue/main.go:683","msg":"RoleTracker: leader election enabled"}
{"level":"error","ts":"2026-11-04T03:47:26.708698305Z","logger":"setup","caller":"kueue/main.go:445","msg":"Skipping admission check controller setup: Provisioning Requests not supported (Possible cause: missing or unsupported cluster-autoscaler)","error":"no matches for kind \"ProvisioningRequest\" in version \"autoscaling.x-k8s.io/v1\"","stacktrace":"main.setupIndexes\n\t/workspace/upstream/kueue/src/cmd/kueue/main.go:445\nmain.main\n\t/workspace/upstream/kueue/src/cmd/kueue/main.go:363\nruntime.main\n\t/usr/lib/golang/src/runtime/proc.go:290"}
{"level":"info","ts":"2026-11-04T03:47:26.761831754Z","logger":"setup","caller":"kueue/main.go:625","msg":"Probe endpoints are configured on healthz and readyz"}
{"level":"info","ts":"2026-11-04T03:47:26.761936072Z","logger":"queue-manager","caller":"queue/manager.go:138","msg":"DRA reconcile channel connected"}
{"level":"info","ts":"2026-11-04T03:47:26.817365403Z","logger":"setup","caller":"kueue/main.go:497","msg":"Skipping provisioning controller setup: Provisioning Requests not supported (Possible cause: missing or unsupported cluster-autoscaler)"}
{"level":"info","ts":"2026-11-04T03:47:26.817741469Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"workload.codeflare.dev/v1beta2, Kind=AppWrapper","path":"/mutate-workload-codeflare-dev-v1beta2-appwrapper"}
{"level":"info","ts":"2026-11-04T03:47:26.818057654Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-workload-codeflare-dev-v1beta2-appwrapper"}
{"level":"info","ts":"2026-11-04T03:47:26.818106292Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"workload.codeflare.dev/v1beta2, Kind=AppWrapper","path":"/validate-workload-codeflare-dev-v1beta2-appwrapper"}
{"level":"info","ts":"2026-11-04T03:47:26.818168416Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-workload-codeflare-dev-v1beta2-appwrapper"}
{"level":"info","ts":"2026-11-04T03:47:26.818337936Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"apps/v1, Kind=Deployment","path":"/mutate-apps-v1-deployment"}
{"level":"info","ts":"2026-11-04T03:47:26.818402792Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-apps-v1-deployment"}
{"level":"info","ts":"2026-11-04T03:47:26.818434045Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"apps/v1, Kind=Deployment","path":"/validate-apps-v1-deployment"}
{"level":"info","ts":"2026-11-04T03:47:26.818477302Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-apps-v1-deployment"}
{"level":"info","ts":"2026-11-04T03:47:26.818555511Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"batch/v1, Kind=Job","path":"/mutate-batch-v1-job"}
{"level":"info","ts":"2026-11-04T03:47:26.818607711Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-batch-v1-job"}
{"level":"info","ts":"2026-11-04T03:47:26.818631298Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"batch/v1, Kind=Job","path":"/validate-batch-v1-job"}
{"level":"info","ts":"2026-11-04T03:47:26.818676558Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-batch-v1-job"}
{"level":"info","ts":"2026-11-04T03:47:26.81873689Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"jobset.x-k8s.io/v1alpha2, Kind=JobSet","path":"/mutate-jobset-x-k8s-io-v1alpha2-jobset"}
{"level":"info","ts":"2026-11-04T03:47:26.818783958Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-jobset-x-k8s-io-v1alpha2-jobset"}
{"level":"info","ts":"2026-11-04T03:47:26.818810032Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"jobset.x-k8s.io/v1alpha2, Kind=JobSet","path":"/validate-jobset-x-k8s-io-v1alpha2-jobset"}
{"level":"info","ts":"2026-11-04T03:47:26.818857875Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-jobset-x-k8s-io-v1alpha2-jobset"}
{"level":"info","ts":"2026-11-04T03:47:26.818913114Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"kubeflow.org/v1, Kind=JAXJob","path":"/mutate-kubeflow-org-v1-jaxjob"}
{"level":"info","ts":"2026-11-04T03:47:26.818964759Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-kubeflow-org-v1-jaxjob"}
{"level":"info","ts":"2026-11-04T03:47:26.818989422Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"kubeflow.org/v1, Kind=JAXJob","path":"/validate-kubeflow-org-v1-jaxjob"}
{"level":"info","ts":"2026-11-04T03:47:26.819034407Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-kubeflow-org-v1-jaxjob"}
{"level":"info","ts":"2026-11-04T03:47:26.819084631Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"kubeflow.org/v1, Kind=PaddleJob","path":"/mutate-kubeflow-org-v1-paddlejob"}
{"level":"info","ts":"2026-11-04T03:47:26.819137604Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-kubeflow-org-v1-paddlejob"}
{"level":"info","ts":"2026-11-04T03:47:26.819161173Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"kubeflow.org/v1, Kind=PaddleJob","path":"/validate-kubeflow-org-v1-paddlejob"}
{"level":"info","ts":"2026-11-04T03:47:26.819224218Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-kubeflow-org-v1-paddlejob"}
{"level":"info","ts":"2026-11-04T03:47:26.819283303Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"kubeflow.org/v1, Kind=PyTorchJob","path":"/mutate-kubeflow-org-v1-pytorchjob"}
{"level":"info","ts":"2026-11-04T03:47:26.819327088Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-kubeflow-org-v1-pytorchjob"}
{"level":"info","ts":"2026-11-04T03:47:26.819351391Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"kubeflow.org/v1, Kind=PyTorchJob","path":"/validate-kubeflow-org-v1-pytorchjob"}
{"level":"info","ts":"2026-11-04T03:47:26.819403047Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-kubeflow-org-v1-pytorchjob"}
{"level":"info","ts":"2026-11-04T03:47:26.819456866Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"kubeflow.org/v1, Kind=TFJob","path":"/mutate-kubeflow-org-v1-tfjob"}
{"level":"info","ts":"2026-11-04T03:47:26.819512111Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-kubeflow-org-v1-tfjob"}
{"level":"info","ts":"2026-11-04T03:47:26.819560562Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"kubeflow.org/v1, Kind=TFJob","path":"/validate-kubeflow-org-v1-tfjob"}
{"level":"info","ts":"2026-11-04T03:47:26.81960788Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-kubeflow-org-v1-tfjob"}
{"level":"info","ts":"2026-11-04T03:47:26.819658113Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"kubeflow.org/v1, Kind=XGBoostJob","path":"/mutate-kubeflow-org-v1-xgboostjob"}
{"level":"info","ts":"2026-11-04T03:47:26.819707008Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-kubeflow-org-v1-xgboostjob"}
{"level":"info","ts":"2026-11-04T03:47:26.819734327Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"kubeflow.org/v1, Kind=XGBoostJob","path":"/validate-kubeflow-org-v1-xgboostjob"}
{"level":"info","ts":"2026-11-04T03:47:26.819772Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-kubeflow-org-v1-xgboostjob"}
{"level":"info","ts":"2026-11-04T03:47:26.819821704Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"kubeflow.org/v2beta1, Kind=MPIJob","path":"/mutate-kubeflow-org-v2beta1-mpijob"}
{"level":"info","ts":"2026-11-04T03:47:26.819855613Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-kubeflow-org-v2beta1-mpijob"}
{"level":"info","ts":"2026-11-04T03:47:26.819876528Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"kubeflow.org/v2beta1, Kind=MPIJob","path":"/validate-kubeflow-org-v2beta1-mpijob"}
{"level":"info","ts":"2026-11-04T03:47:26.819913216Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-kubeflow-org-v2beta1-mpijob"}
{"level":"info","ts":"2026-11-04T03:47:26.820006411Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"/v1, Kind=Pod","path":"/mutate--v1-pod"}
{"level":"info","ts":"2026-11-04T03:47:26.820047513Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate--v1-pod"}
{"level":"info","ts":"2026-11-04T03:47:26.820069757Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"/v1, Kind=Pod","path":"/validate--v1-pod"}
{"level":"info","ts":"2026-11-04T03:47:26.820122719Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate--v1-pod"}
{"level":"info","ts":"2026-11-04T03:47:26.820179069Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"leaderworkerset.x-k8s.io/v1, Kind=LeaderWorkerSet","path":"/mutate-leaderworkerset-x-k8s-io-v1-leaderworkerset"}
{"level":"info","ts":"2026-11-04T03:47:26.820244222Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-leaderworkerset-x-k8s-io-v1-leaderworkerset"}
{"level":"info","ts":"2026-11-04T03:47:26.820271738Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"leaderworkerset.x-k8s.io/v1, Kind=LeaderWorkerSet","path":"/validate-leaderworkerset-x-k8s-io-v1-leaderworkerset"}
{"level":"info","ts":"2026-11-04T03:47:26.820309278Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-leaderworkerset-x-k8s-io-v1-leaderworkerset"}
{"level":"info","ts":"2026-11-04T03:47:26.820358939Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"ray.io/v1, Kind=RayCluster","path":"/mutate-ray-io-v1-raycluster"}
{"level":"info","ts":"2026-11-04T03:47:26.820392777Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-ray-io-v1-raycluster"}
{"level":"info","ts":"2026-11-04T03:47:26.820413737Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"ray.io/v1, Kind=RayCluster","path":"/validate-ray-io-v1-raycluster"}
{"level":"info","ts":"2026-11-04T03:47:26.820446637Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-ray-io-v1-raycluster"}
{"level":"info","ts":"2026-11-04T03:47:26.820497864Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"ray.io/v1, Kind=RayJob","path":"/mutate-ray-io-v1-rayjob"}
{"level":"info","ts":"2026-11-04T03:47:26.820552082Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-ray-io-v1-rayjob"}
{"level":"info","ts":"2026-11-04T03:47:26.820586436Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"ray.io/v1, Kind=RayJob","path":"/validate-ray-io-v1-rayjob"}
{"level":"info","ts":"2026-11-04T03:47:26.820646529Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-ray-io-v1-rayjob"}
{"level":"info","ts":"2026-11-04T03:47:26.820722968Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"ray.io/v1, Kind=RayService","path":"/mutate-ray-io-v1-rayservice"}
{"level":"info","ts":"2026-11-04T03:47:26.820795731Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-ray-io-v1-rayservice"}
{"level":"info","ts":"2026-11-04T03:47:26.820819436Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"ray.io/v1, Kind=RayService","path":"/validate-ray-io-v1-rayservice"}
{"level":"info","ts":"2026-11-04T03:47:26.820854147Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-ray-io-v1-rayservice"}
{"level":"info","ts":"2026-11-04T03:47:26.820903596Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"sparkoperator.k8s.io/v1beta2, Kind=SparkApplication","path":"/mutate-sparkoperator-k8s-io-v1beta2-sparkapplication"}
{"level":"info","ts":"2026-11-04T03:47:26.820936499Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-sparkoperator-k8s-io-v1beta2-sparkapplication"}
{"level":"info","ts":"2026-11-04T03:47:26.820959207Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"sparkoperator.k8s.io/v1beta2, Kind=SparkApplication","path":"/validate-sparkoperator-k8s-io-v1beta2-sparkapplication"}
{"level":"info","ts":"2026-11-04T03:47:26.821025558Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-sparkoperator-k8s-io-v1beta2-sparkapplication"}
{"level":"info","ts":"2026-11-04T03:47:26.821077499Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"apps/v1, Kind=StatefulSet","path":"/mutate-apps-v1-statefulset"}
{"level":"info","ts":"2026-11-04T03:47:26.821112458Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-apps-v1-statefulset"}
{"level":"info","ts":"2026-11-04T03:47:26.821133187Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"apps/v1, Kind=StatefulSet","path":"/validate-apps-v1-statefulset"}
{"level":"info","ts":"2026-11-04T03:47:26.821178072Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-apps-v1-statefulset"}
{"level":"info","ts":"2026-11-04T03:47:26.821526297Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","path":"/mutate-trainer-kubeflow-org-v1alpha1-trainjob"}
{"level":"info","ts":"2026-11-04T03:47:26.821582794Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-trainer-kubeflow-org-v1alpha1-trainjob"}
{"level":"info","ts":"2026-11-04T03:47:26.821606834Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"trainer.kubeflow.org/v1alpha1, Kind=TrainJob","path":"/validate-trainer-kubeflow-org-v1alpha1-trainjob"}
{"level":"info","ts":"2026-11-04T03:47:26.821645099Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-trainer-kubeflow-org-v1alpha1-trainjob"}
{"level":"info","ts":"2026-11-04T03:47:26.821704139Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"kueue.x-k8s.io/v1beta2, Kind=Workload","path":"/mutate-kueue-x-k8s-io-v1beta2-workload"}
{"level":"info","ts":"2026-11-04T03:47:26.821739023Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-kueue-x-k8s-io-v1beta2-workload"}
{"level":"info","ts":"2026-11-04T03:47:26.821760377Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"kueue.x-k8s.io/v1beta2, Kind=Workload","path":"/validate-kueue-x-k8s-io-v1beta2-workload"}
{"level":"info","ts":"2026-11-04T03:47:26.821796864Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-kueue-x-k8s-io-v1beta2-workload"}
{"level":"info","ts":"2026-11-04T03:47:26.821866903Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/convert"}
{"level":"info","ts":"2026-11-04T03:47:26.821875568Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:319","msg":"Conversion webhook enabled","GVK":"kueue.x-k8s.io/v1beta2, Kind=Workload"}
{"level":"info","ts":"2026-11-04T03:47:26.821896676Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"kueue.x-k8s.io/v1beta2, Kind=ResourceFlavor","path":"/mutate-kueue-x-k8s-io-v1beta2-resourceflavor"}
{"level":"info","ts":"2026-11-04T03:47:26.821932411Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-kueue-x-k8s-io-v1beta2-resourceflavor"}
{"level":"info","ts":"2026-11-04T03:47:26.821952821Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"kueue.x-k8s.io/v1beta2, Kind=ResourceFlavor","path":"/validate-kueue-x-k8s-io-v1beta2-resourceflavor"}
{"level":"info","ts":"2026-11-04T03:47:26.821988893Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-kueue-x-k8s-io-v1beta2-resourceflavor"}
{"level":"info","ts":"2026-11-04T03:47:26.822035153Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:221","msg":"Registering a mutating webhook","GVK":"kueue.x-k8s.io/v1beta2, Kind=ClusterQueue","path":"/mutate-kueue-x-k8s-io-v1beta2-clusterqueue"}
{"level":"info","ts":"2026-11-04T03:47:26.822071878Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/mutate-kueue-x-k8s-io-v1beta2-clusterqueue"}
{"level":"info","ts":"2026-11-04T03:47:26.822093878Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"kueue.x-k8s.io/v1beta2, Kind=ClusterQueue","path":"/validate-kueue-x-k8s-io-v1beta2-clusterqueue"}
{"level":"info","ts":"2026-11-04T03:47:26.822129808Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-kueue-x-k8s-io-v1beta2-clusterqueue"}
{"level":"info","ts":"2026-11-04T03:47:26.82216153Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:319","msg":"Conversion webhook enabled","GVK":"kueue.x-k8s.io/v1beta2, Kind=ClusterQueue"}
{"level":"info","ts":"2026-11-04T03:47:26.822183626Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:268","msg":"Registering a validating webhook","GVK":"kueue.x-k8s.io/v1beta2, Kind=Cohort","path":"/validate-kueue-x-k8s-io-v1beta2-cohort"}
{"level":"info","ts":"2026-11-04T03:47:26.822235805Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:183","msg":"Registering webhook","path":"/validate-kueue-x-k8s-io-v1beta2-cohort"}
{"level":"info","ts":"2026-11-04T03:47:26.822266661Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:319","msg":"Conversion webhook enabled","GVK":"kueue.x-k8s.io/v1beta2, Kind=Cohort"}
{"level":"info","ts":"2026-11-04T03:47:26.822297853Z","logger":"controller-runtime.builder","caller":"builder/webhook.go:319","msg":"Conversion webhook enabled","GVK":"kueue.x-k8s.io/v1beta2, Kind=LocalQueue"}
{"level":"info","ts":"2026-11-04T03:47:26.822320344Z","logger":"setup","caller":"kueue/main.go:430","msg":"Starting manager"}
{"level":"info","ts":"2026-11-04T03:47:26.822377847Z","logger":"controller-runtime.metrics","caller":"server/server.go:208","msg":"Starting metrics server"}
{"level":"info","ts":"2026-11-04T03:47:26.822439581Z","caller":"manager/server.go:83","msg":"starting server","name":"health probe","addr":"[::]:8081"}
{"level":"info","ts":"2026-11-04T03:47:26.822486153Z","logger":"controller-runtime.metrics","caller":"server/server.go:247","msg":"Serving metrics server","bindAddress":":8443","secure":true}
{"level":"info","ts":"2026-11-04T03:47:26.822502612Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:191","msg":"Starting webhook server"}
{"level":"info","ts":"2026-11-04T03:47:26.822966125Z","logger":"controller-runtime.certwatcher","caller":"certwatcher/certwatcher.go:214","msg":"Updated current TLS certificate","cert":"/tmp/k8s-webhook-server/serving-certs/tls.crt","key":"/tmp/k8s-webhook-server/serving-certs/tls.key"}
{"level":"info","ts":"2026-11-04T03:47:26.823087936Z","logger":"controller-runtime.webhook","caller":"webhook/server.go:242","msg":"Serving webhook server","host":"","port":9443}
{"level":"info","ts":"2026-11-04T03:47:26.823182568Z","logger":"controller-runtime.certwatcher","caller":"certwatcher/certwatcher.go:136","msg":"Starting certificate poll+watcher","cert":"/tmp/k8s-webhook-server/serving-certs/tls.crt","key":"/tmp/k8s-webhook-server/serving-certs/tls.key","interval":10}
{"level":"Level(-2)","ts":"2026-11-04T03:47:26.879406455Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1beta2.MultiKueueCluster","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-04T03:47:26.879527239Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1beta2.MultiKueueConfig","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-04T03:47:26.879530062Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1.LimitRange","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-04T03:47:26.879584666Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1alpha1.ClusterTrainingRuntime","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-04T03:47:26.879703188Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1beta2.LocalQueue","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-04T03:47:26.880868233Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1alpha1.TrainingRuntime","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-04T03:47:26.881487285Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1alpha1.TrainJob","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-04T03:47:26.881495138Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1.Node","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-04T03:47:26.881672541Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1beta2.Workload","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-04T03:47:26.931005617Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1beta2.AdmissionCheck","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-04T03:47:26.931102401Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1beta2.ResourceFlavor","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-04T03:47:27.037738305Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1.Pod","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
I1104 03:47:27.081927       1 plugins.go:157] Loaded 1 mutating admission controller(s) successfully in the following order: NamespaceLifecycle.
I1104 03:47:27.085078       1 handler.go:304] Adding GroupVersion visibility.kueue.x-k8s.io v1beta2 to ResourceManager
I1104 03:47:27.085779       1 handler.go:304] Adding GroupVersion visibility.kueue.x-k8s.io v1beta1 to ResourceManager
{"level":"info","ts":"2026-11-04T03:47:27.123807124Z","logger":"controller-runtime.certwatcher","caller":"certwatcher/certwatcher.go:136","msg":"Starting certificate poll+watcher","cert":"/etc/kueue/metrics/certs/tls.crt","key":"/etc/kueue/metrics/certs/tls.key","interval":10}
{"level":"info","ts":"2026-11-04T03:47:27.123943199Z","logger":"admissioncheck-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"channel source: 0x20ae753533b0"}
{"level":"info","ts":"2026-11-04T03:47:27.123988287Z","logger":"admissioncheck-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1beta2.AdmissionCheck"}
I1104 03:47:27.123985       1 leaderelection.go:258] "Attempting to acquire leader lease..." lock="openshift-kueue-operator/c1f6bfd2.kueue.x-k8s.io"
{"level":"info","ts":"2026-11-04T03:47:27.124062357Z","logger":"ElasticJobUngater","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1beta2.Workload"}
{"level":"info","ts":"2026-11-04T03:47:27.124086028Z","logger":"clusterqueue-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1.Namespace"}
{"level":"info","ts":"2026-11-04T03:47:27.124094016Z","logger":"clusterqueue-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1beta2.ClusterQueue"}
{"level":"info","ts":"2026-11-04T03:47:27.124062397Z","logger":"clusterqueue-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"channel source: 0x20ae750b6000"}
{"level":"info","ts":"2026-11-04T03:47:27.124103585Z","logger":"ElasticJobUngater","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1.Pod"}
{"level":"info","ts":"2026-11-04T03:47:27.123943197Z","logger":"workloadpriorityclass-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1beta2.WorkloadPriorityClass"}
{"level":"info","ts":"2026-11-04T03:47:27.124165455Z","logger":"localqueue-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1beta2.ClusterQueue"}
{"level":"info","ts":"2026-11-04T03:47:27.124184203Z","logger":"localqueue-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"channel source: 0x20ae75353c70"}
{"level":"info","ts":"2026-11-04T03:47:27.124196462Z","logger":"tas-topology-controller","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1beta2.Topology"}
{"level":"info","ts":"2026-11-04T03:47:27.124234347Z","logger":"localqueue-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1beta2.LocalQueue"}
{"level":"info","ts":"2026-11-04T03:47:27.124320113Z","logger":"resourceflavor-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"channel source: 0x20ae753532d0"}
{"level":"info","ts":"2026-11-04T03:47:27.124374426Z","logger":"cohort-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1beta2.Cohort"}
{"level":"info","ts":"2026-11-04T03:47:27.124240965Z","logger":"tas-topology-controller","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1beta2.ResourceFlavor"}
{"level":"info","ts":"2026-11-04T03:47:27.124352519Z","logger":"cohort-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"channel source: 0x20ae75353dc0"}
{"level":"info","ts":"2026-11-04T03:47:27.124335504Z","logger":"resourceflavor-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1beta2.ResourceFlavor"}
{"level":"info","ts":"2026-11-04T03:47:27.124461534Z","logger":"tas-topology-ungater","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1beta2.Workload"}
{"level":"info","ts":"2026-11-04T03:47:27.124466387Z","logger":"tas-topology-ungater","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1.Pod"}
{"level":"info","ts":"2026-11-04T03:47:27.124602962Z","logger":"workload-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1.RuntimeClass"}
{"level":"info","ts":"2026-11-04T03:47:27.124623002Z","logger":"tas-resource-flavor-controller","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"channel source: 0x20ae75285030"}
{"level":"info","ts":"2026-11-04T03:47:27.124606781Z","logger":"workload-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1.LimitRange"}
{"level":"info","ts":"2026-11-04T03:47:27.124634327Z","logger":"workload-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1beta2.ClusterQueue"}
{"level":"info","ts":"2026-11-04T03:47:27.124635166Z","logger":"workload-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1beta2.LocalQueue"}
{"level":"info","ts":"2026-11-04T03:47:27.124658886Z","logger":"workload-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1beta2.Workload"}
{"level":"info","ts":"2026-11-04T03:47:27.124607536Z","logger":"workload-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"channel source: 0x20ae7547f8f0"}
{"level":"info","ts":"2026-11-04T03:47:27.124726455Z","logger":"tas-resource-flavor-controller","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1beta2.ResourceFlavor"}
{"level":"info","ts":"2026-11-04T03:47:27.124735913Z","logger":"tas-non-tas-usage-controller","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1.Pod"}
{"level":"info","ts":"2026-11-04T03:47:27.124788821Z","logger":"tas-node-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1.Node"}
{"level":"info","ts":"2026-11-04T03:47:27.124820443Z","logger":"tas-node-reconciler","caller":"controller/controller.go:369","msg":"Starting EventSource","replica-role":"follower","source":"kind source: *v1.Pod"}
I1104 03:47:27.144227       1 requestheader_controller.go:180] Starting RequestHeaderAuthRequestController
I1104 03:47:27.144241       1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::client-ca-file"
I1104 03:47:27.144241       1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file"
I1104 03:47:27.144260       1 shared_informer.go:402] "Waiting for caches to sync"
I1104 03:47:27.144261       1 shared_informer.go:402] "Waiting for caches to sync"
I1104 03:47:27.144263       1 shared_informer.go:402] "Waiting for caches to sync"
I1104 03:47:27.144612       1 dynamic_serving_content.go:135] "Starting controller" name="serving-cert::/visibility/tls.crt::/visibility/tls.key"
I1104 03:47:27.145018       1 secure_serving.go:214] Serving securely on [::]:8082
I1104 03:47:27.145059       1 tlsconfig.go:243] "Starting DynamicServingCertificateController"
I1104 03:47:27.145099       1 apf_controller.go:377] Starting API Priority and Fairness config controller
{"level":"Level(-2)","ts":"2026-11-04T03:47:27.178928395Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1.RuntimeClass","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-04T03:47:27.179109203Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1beta2.WorkloadPriorityClass","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-04T03:47:27.179131954Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1beta2.Cohort","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-04T03:47:27.179149362Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1beta2.Topology","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-04T03:47:27.17931619Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1beta2.ClusterQueue","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"Level(-2)","ts":"2026-11-04T03:47:27.181333092Z","logger":"controller-runtime.cache","caller":"cache/reflector.go:507","msg":"Caches populated","type":"*v1.Namespace","reflector":"sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:128"}
{"level":"info","ts":"2026-11-04T03:47:27.225185274Z","logger":"resourceflavor-reconciler","caller":"controller/controller.go:302","msg":"Starting Controller","replica-role":"follower"}
{"level":"info","ts":"2026-11-04T03:47:27.225199781Z","logger":"ElasticJobUngater","caller":"controller/controller.go:302","msg":"Starting Controller","replica-role":"follower"}
{"level":"info","ts":"2026-11-04T03:47:27.225230957Z","logger":"tas-resource-flavor-controller","caller":"controller/controller.go:302","msg":"Starting Controller","replica-role":"follower"}
{"level":"info","ts":"2026-11-04T03:47:27.225248279Z","logger":"resourceflavor-reconciler","caller":"controller/controller.go:305","msg":"Starting workers","replica-role":"follower","worker count":1}
{"level":"info","ts":"2026-11-04T03:47:27.225256501Z","logger":"ElasticJobUngater","caller":"controller/controller.go:305","msg":"Starting workers","replica-role":"follower","worker count":5}
{"level":"info","ts":"2026-11-04T03:47:27.225262439Z","logger":"tas-resource-flavor-controller","caller":"controller/controller.go:305","msg":"Starting workers","replica-role":"follower","worker count":1}
{"level":"info","ts":"2026-11-04T03:47:27.225302746Z","logger":"tas-topology-ungater","caller":"controller/controller.go:302","msg":"Starting Controller","replica-role":"follower"}
{"level":"info","ts":"2026-11-04T03:47:27.22538681Z","logger":"tas-topology-ungater","caller":"controller/controller.go:305","msg":"Starting workers","replica-role":"follower","worker count":5}
{"level":"info","ts":"2026-11-04T03:47:27.225374647Z","logger":"tas-node-reconciler","caller":"controller/controller.go:302","msg":"Starting Controller","replica-role":"follower"}
{"level":"info","ts":"2026-11-04T03:47:27.225440172Z","logger":"tas-node-reconciler","caller":"controller/controller.go:305","msg":"Starting workers","replica-role":"follower","worker count":1}
{"level":"info","ts":"2026-11-04T03:47:27.225442707Z","logger":"admissioncheck-reconciler","caller":"controller/controller.go:302","msg":"Starting Controller","replica-role":"follower"}
{"level":"info","ts":"2026-11-04T03:47:27.225464464Z","logger":"admissioncheck-reconciler","caller":"controller/controller.go:305","msg":"Starting workers","replica-role":"follower","worker count":1}
{"level":"info","ts":"2026-11-04T03:47:27.225200285Z","logger":"tas-non-tas-usage-controller","caller":"controller/controller.go:302","msg":"Starting Controller","replica-role":"follower"}
{"level":"info","ts":"2026-11-04T03:47:27.225702386Z","logger":"tas-non-tas-usage-controller","caller":"controller/controller.go:305","msg":"Starting workers","replica-role":"follower","worker count":5}
I1104 03:47:27.245017       1 shared_informer.go:409] "Caches are synced"
I1104 03:47:27.245108       1 shared_informer.go:409] "Caches are synced"
I1104 03:47:27.245121       1 shared_informer.go:409] "Caches are synced"
I1104 03:47:27.245243       1 apf_controller.go:382] Running API Priority and Fairness config worker
I1104 03:47:27.245256       1 apf_controller.go:385] Running API Priority and Fairness periodic rebalancing process
{"level":"info","ts":"2026-11-04T03:47:27.326291034Z","logger":"tas-topology-controller","caller":"controller/controller.go:302","msg":"Starting Controller","replica-role":"follower"}
{"level":"info","ts":"2026-11-04T03:47:27.32633664Z","logger":"tas-topology-controller","caller":"controller/controller.go:305","msg":"Starting workers","replica-role":"follower","worker count":1}
{"level":"info","ts":"2026-11-04T03:47:27.326343773Z","logger":"clusterqueue-reconciler","caller":"controller/controller.go:302","msg":"Starting Controller","replica-role":"follower"}
{"level":"info","ts":"2026-11-04T03:47:27.326301583Z","logger":"workloadpriorityclass-reconciler","caller":"controller/controller.go:302","msg":"Starting Controller","replica-role":"follower"}
{"level":"info","ts":"2026-11-04T03:47:27.326355958Z","logger":"clusterqueue-reconciler","caller":"controller/controller.go:305","msg":"Starting workers","replica-role":"follower","worker count":1}
{"level":"info","ts":"2026-11-04T03:47:27.326359289Z","logger":"workloadpriorityclass-reconciler","caller":"controller/controller.go:305","msg":"Starting workers","replica-role":"follower","worker count":1}
{"level":"info","ts":"2026-11-04T03:47:27.326377096Z","logger":"cohort-reconciler","caller":"controller/controller.go:302","msg":"Starting Controller","replica-role":"follower"}
{"level":"info","ts":"2026-11-04T03:47:27.326391118Z","logger":"cohort-reconciler","caller":"controller/controller.go:305","msg":"Starting workers","replica-role":"follower","worker count":1}
{"level":"info","ts":"2026-11-04T03:47:27.326406205Z","logger":"localqueue-reconciler","caller":"controller/controller.go:302","msg":"Starting Controller","replica-role":"follower"}
{"level":"info","ts":"2026-11-04T03:47:27.326417928Z","logger":"localqueue-reconciler","caller":"controller/controller.go:305","msg":"Starting workers","replica-role":"follower","worker count":1}
{"level":"info","ts":"2026-11-04T03:47:27.32642213Z","logger":"workload-reconciler","caller":"controller/controller.go:302","msg":"Starting Controller","replica-role":"follower"}
{"level":"info","ts":"2026-11-04T03:47:27.326429898Z","logger":"workload-reconciler","caller":"controller/controller.go:305","msg":"Starting workers","replica-role":"follower","worker count":5}
{"level":"Level(-2)","ts":"2026-11-04T04:23:21.493993473Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:141","msg":"ResourceFlavor create event","replica-role":"follower","resourceFlavor":{"name":"rf-pcwdn"}}
{"level":"Level(-2)","ts":"2026-11-04T04:23:21.869883403Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:351","msg":"ClusterQueue create event","replica-role":"follower","clusterQueue":{"name":"cq-swpxd"}}
{"level":"Level(-2)","ts":"2026-11-04T04:23:21.888254281Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:240","msg":"LocalQueue create event","replica-role":"follower","localQueue":{"name":"lq-d7rrf","namespace":"test-ns-w5fhp"}}
{"level":"Level(-2)","ts":"2026-11-04T04:23:21.925276538Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"follower","clusterQueue":{"name":"cq-swpxd"}}
{"level":"Level(-2)","ts":"2026-11-04T04:23:21.942647611Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"follower","localQueue":{"name":"lq-d7rrf","namespace":"test-ns-w5fhp"}}
{"level":"Level(-2)","ts":"2026-11-04T04:23:22.046666297Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"follower","localQueue":{"name":"lq-d7rrf","namespace":"test-ns-w5fhp"}}
{"level":"Level(-2)","ts":"2026-11-04T04:23:23.256558249Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1276","msg":"Workload create event","replica-role":"follower","workload":{"name":"pod-dw-deployment-h4b5d-5f87cf994b-hh8n4-9866a","namespace":"test-ns-w5fhp"},"queue":"lq-d7rrf","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:23.418228454Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"follower","workload":{"name":"pod-dw-deployment-h4b5d-5f87cf994b-hh8n4-9866a","namespace":"test-ns-w5fhp"},"queue":"lq-d7rrf","status":"admitted","prevStatus":"pending","clusterQueue":"cq-swpxd"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:24.355722326Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"follower","localQueue":{"name":"lq-d7rrf","namespace":"test-ns-w5fhp"}}
{"level":"Level(-2)","ts":"2026-11-04T04:23:24.35870429Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"follower","clusterQueue":{"name":"cq-swpxd"}}
{"level":"info","ts":"2026-11-04T04:23:38.030953673Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:117","msg":"Validating create","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"ycbd600a3ac3","namespace":"test-ns-w5fhp"},"namespace":"test-ns-w5fhp","name":"ycbd600a3ac3","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:redhat-ods-applications:trainer-e2e","requestID":"f5f70e83-52b7-42d6-9d01-e907b8014012","replica-role":"follower"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:38.423180061Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1276","msg":"Workload create event","replica-role":"follower","workload":{"name":"trainjob-ycbd600a3ac3-991e3","namespace":"test-ns-w5fhp"},"queue":"lq-d7rrf","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:38.534688806Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"follower","workload":{"name":"trainjob-ycbd600a3ac3-991e3","namespace":"test-ns-w5fhp"},"queue":"lq-d7rrf","status":"admitted","prevStatus":"pending","clusterQueue":"cq-swpxd"}
{"level":"info","ts":"2026-11-04T04:23:38.858581251Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"ycbd600a3ac3","namespace":"test-ns-w5fhp"},"namespace":"test-ns-w5fhp","name":"ycbd600a3ac3","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"fc7c7d19-c92f-44e8-b492-3e34b98c31e1","replica-role":"follower"}
{"level":"info","ts":"2026-11-04T04:23:39.25923028Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"ycbd600a3ac3","namespace":"test-ns-w5fhp"},"namespace":"test-ns-w5fhp","name":"ycbd600a3ac3","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"a3b57476-9216-4eeb-a974-f22662b6919d","replica-role":"follower"}
{"level":"info","ts":"2026-11-04T04:23:39.436278704Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"ycbd600a3ac3","namespace":"test-ns-w5fhp"},"namespace":"test-ns-w5fhp","name":"ycbd600a3ac3","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"43e12ea4-bd9e-497f-a981-44fe28a2d49e","replica-role":"follower"}
{"level":"Level(-2)","ts":"2026-11-04T04:23:39.476551767Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"follower","localQueue":{"name":"lq-d7rrf","namespace":"test-ns-w5fhp"}}
{"level":"Level(-2)","ts":"2026-11-04T04:23:39.479638446Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"follower","clusterQueue":{"name":"cq-swpxd"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:14.491369003Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"follower","workload":{"name":"trainjob-ycbd600a3ac3-991e3","namespace":"test-ns-w5fhp"},"queue":"lq-d7rrf","status":"finished","prevStatus":"admitted","clusterQueue":"cq-swpxd"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:14.782186216Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"follower","workload":{"name":"trainjob-ycbd600a3ac3-991e3","namespace":"test-ns-w5fhp"},"queue":"lq-d7rrf","status":"finished","clusterQueue":"cq-swpxd"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:15.546853838Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"follower","localQueue":{"name":"lq-d7rrf","namespace":"test-ns-w5fhp"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:15.549768119Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"follower","clusterQueue":{"name":"cq-swpxd"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:16.404609805Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1312","msg":"Workload delete event","replica-role":"follower","workload":{"name":"trainjob-ycbd600a3ac3-991e3","namespace":"test-ns-w5fhp"},"queue":"lq-d7rrf","status":"finished"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:18.175144064Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"follower","clusterQueue":{"name":"cq-swpxd"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:18.192769127Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:171","msg":"ResourceFlavor update event","replica-role":"follower","resourceFlavor":{"name":"rf-pcwdn"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:18.2362604Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"follower","clusterQueue":{"name":"cq-swpxd"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:18.291026079Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"follower","localQueue":{"name":"lq-d7rrf","namespace":"test-ns-w5fhp"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:18.525640416Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:141","msg":"ResourceFlavor create event","replica-role":"follower","resourceFlavor":{"name":"rf-rt7ms"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:18.652777858Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:351","msg":"ClusterQueue create event","replica-role":"follower","clusterQueue":{"name":"cq-c22x6"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:18.669569854Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:240","msg":"LocalQueue create event","replica-role":"follower","localQueue":{"name":"lq-bl6f8","namespace":"test-ns-dqfgt"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:18.705639433Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"follower","clusterQueue":{"name":"cq-c22x6"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:18.721780624Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"follower","localQueue":{"name":"lq-bl6f8","namespace":"test-ns-dqfgt"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:18.776127623Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"follower","localQueue":{"name":"lq-bl6f8","namespace":"test-ns-dqfgt"}}
{"level":"info","ts":"2026-11-04T04:24:18.792877381Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:117","msg":"Validating create","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-preemption-trainjob-t7bj6","namespace":"test-ns-dqfgt"},"namespace":"test-ns-dqfgt","name":"test-preemption-trainjob-t7bj6","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:admin","requestID":"bd0fdd34-ad1e-4347-a981-c45352bf14ac","replica-role":"follower"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:19.176368232Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1276","msg":"Workload create event","replica-role":"follower","workload":{"name":"trainjob-test-preemption-trainjob-t7bj6-a698f","namespace":"test-ns-dqfgt"},"queue":"lq-bl6f8","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:19.288305243Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"follower","workload":{"name":"trainjob-test-preemption-trainjob-t7bj6-a698f","namespace":"test-ns-dqfgt"},"queue":"lq-bl6f8","status":"admitted","prevStatus":"pending","clusterQueue":"cq-c22x6"}
{"level":"info","ts":"2026-11-04T04:24:19.864578252Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-preemption-trainjob-t7bj6","namespace":"test-ns-dqfgt"},"namespace":"test-ns-dqfgt","name":"test-preemption-trainjob-t7bj6","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"4d801d9c-a52c-459b-b11f-8008312a607e","replica-role":"follower"}
{"level":"info","ts":"2026-11-04T04:24:20.033452236Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-preemption-trainjob-t7bj6","namespace":"test-ns-dqfgt"},"namespace":"test-ns-dqfgt","name":"test-preemption-trainjob-t7bj6","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"dcb286d6-bd08-4508-a79d-d15d3d5b3a3b","replica-role":"follower"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:20.23150538Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"follower","localQueue":{"name":"lq-bl6f8","namespace":"test-ns-dqfgt"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:20.23162535Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"follower","clusterQueue":{"name":"cq-c22x6"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:22.016334098Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"follower","workload":{"name":"trainjob-test-preemption-trainjob-t7bj6-a698f","namespace":"test-ns-dqfgt"},"queue":"lq-bl6f8","status":"admitted","clusterQueue":"cq-c22x6"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:22.016392721Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1381","msg":"Workload will not be queued because the workload is not active","replica-role":"follower","workload":{"name":"trainjob-test-preemption-trainjob-t7bj6-a698f","namespace":"test-ns-dqfgt"},"queue":"lq-bl6f8","status":"admitted","clusterQueue":"cq-c22x6"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:22.127200299Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"follower","workload":{"name":"trainjob-test-preemption-trainjob-t7bj6-a698f","namespace":"test-ns-dqfgt"},"queue":"lq-bl6f8","status":"admitted","clusterQueue":"cq-c22x6"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:22.127283174Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1381","msg":"Workload will not be queued because the workload is not active","replica-role":"follower","workload":{"name":"trainjob-test-preemption-trainjob-t7bj6-a698f","namespace":"test-ns-dqfgt"},"queue":"lq-bl6f8","status":"admitted","clusterQueue":"cq-c22x6"}
{"level":"info","ts":"2026-11-04T04:24:22.437590608Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-preemption-trainjob-t7bj6","namespace":"test-ns-dqfgt"},"namespace":"test-ns-dqfgt","name":"test-preemption-trainjob-t7bj6","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"abf4cc30-190f-4298-8bb0-56fef9d40ecb","replica-role":"follower"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.072177055Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"follower","localQueue":{"name":"lq-bl6f8","namespace":"test-ns-dqfgt"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.074932664Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"follower","clusterQueue":{"name":"cq-c22x6"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.081969463Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"follower","clusterQueue":{"name":"cq-c22x6"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.097469385Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:171","msg":"ResourceFlavor update event","replica-role":"follower","resourceFlavor":{"name":"rf-rt7ms"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.19444733Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:376","msg":"ClusterQueue delete event","replica-role":"follower","clusterQueue":{"name":"cq-c22x6"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.19452547Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:385","msg":"Cleared resource metrics for deleted ClusterQueue.","replica-role":"follower","clusterQueue":{"name":"cq-c22x6"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.245938318Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"follower","localQueue":{"name":"lq-bl6f8","namespace":"test-ns-dqfgt"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.293639704Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:159","msg":"ResourceFlavor delete event","replica-role":"follower","resourceFlavor":{"name":"rf-rt7ms"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.382615722Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:141","msg":"ResourceFlavor create event","replica-role":"follower","resourceFlavor":{"name":"rf-4g785"}}
{"level":"info","ts":"2026-11-04T04:24:23.395861768Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-preemption-trainjob-t7bj6","namespace":"test-ns-dqfgt"},"namespace":"test-ns-dqfgt","name":"test-preemption-trainjob-t7bj6","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"81b75c36-9848-4063-a823-79dbd8924fe2","replica-role":"follower"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.512526597Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:351","msg":"ClusterQueue create event","replica-role":"follower","clusterQueue":{"name":"cq-44w2t"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.541500331Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:240","msg":"LocalQueue create event","replica-role":"follower","localQueue":{"name":"lq-48blb","namespace":"test-ns-whn67"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.584652772Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"follower","clusterQueue":{"name":"cq-44w2t"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.585289435Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"follower","workload":{"name":"trainjob-test-preemption-trainjob-t7bj6-a698f","namespace":"test-ns-dqfgt"},"queue":"lq-bl6f8","status":"pending","prevStatus":"admitted","prevClusterQueue":"cq-c22x6"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.585330512Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1381","msg":"Workload will not be queued because the workload is not active","replica-role":"follower","workload":{"name":"trainjob-test-preemption-trainjob-t7bj6-a698f","namespace":"test-ns-dqfgt"},"queue":"lq-bl6f8","status":"pending","prevStatus":"admitted","prevClusterQueue":"cq-c22x6"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.595607886Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"follower","localQueue":{"name":"lq-48blb","namespace":"test-ns-whn67"}}
{"level":"info","ts":"2026-11-04T04:24:23.672181655Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:117","msg":"Validating create","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-runtimepatches-trainjob-w2wqc","namespace":"test-ns-whn67"},"namespace":"test-ns-whn67","name":"test-runtimepatches-trainjob-w2wqc","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:admin","requestID":"c983ab09-721b-4490-896c-5dd4c54aaba9","replica-role":"follower"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.698288883Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"follower","workload":{"name":"trainjob-test-preemption-trainjob-t7bj6-a698f","namespace":"test-ns-dqfgt"},"queue":"lq-bl6f8","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.698333583Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1381","msg":"Workload will not be queued because the workload is not active","replica-role":"follower","workload":{"name":"trainjob-test-preemption-trainjob-t7bj6-a698f","namespace":"test-ns-dqfgt"},"queue":"lq-bl6f8","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:23.700257342Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"follower","localQueue":{"name":"lq-48blb","namespace":"test-ns-whn67"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:24.10871553Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1276","msg":"Workload create event","replica-role":"follower","workload":{"name":"trainjob-test-runtimepatches-trainjob-w2wqc-fbb90","namespace":"test-ns-whn67"},"queue":"lq-48blb","status":"pending"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:24.219264275Z","logger":"workload-reconciler","caller":"core/workload_controller.go:1369","msg":"Workload update event","replica-role":"follower","workload":{"name":"trainjob-test-runtimepatches-trainjob-w2wqc-fbb90","namespace":"test-ns-whn67"},"queue":"lq-48blb","status":"admitted","prevStatus":"pending","clusterQueue":"cq-44w2t"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:24.814330261Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"follower","clusterQueue":{"name":"cq-44w2t"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:24.830016539Z","logger":"resourceflavor-reconciler","caller":"core/resourceflavor_controller.go:171","msg":"ResourceFlavor update event","replica-role":"follower","resourceFlavor":{"name":"rf-4g785"}}
{"level":"Level(-2)","ts":"2026-11-04T04:24:24.869681027Z","logger":"cluster-queue-reconciler","caller":"core/clusterqueue_controller.go:392","msg":"ClusterQueue update event","replica-role":"follower","clusterQueue":{"name":"cq-44w2t"}}
{"level":"info","ts":"2026-11-04T04:24:24.895200383Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-runtimepatches-trainjob-w2wqc","namespace":"test-ns-whn67"},"namespace":"test-ns-whn67","name":"test-runtimepatches-trainjob-w2wqc","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"b16463fd-f7bb-4ade-a6ea-4dbe1077ba09","replica-role":"follower"}
{"level":"Level(-2)","ts":"2026-11-04T04:24:24.924270842Z","logger":"localqueue-reconciler","caller":"core/localqueue_controller.go:286","msg":"Queue update event","replica-role":"follower","localQueue":{"name":"lq-48blb","namespace":"test-ns-whn67"}}
{"level":"info","ts":"2026-11-04T04:24:25.06638951Z","logger":"admission.trainjob-webhook","caller":"trainjob/trainjob_webhook.go:130","msg":"Validating update","webhookGroup":"trainer.kubeflow.org","webhookKind":"TrainJob","object":{"name":"test-runtimepatches-trainjob-w2wqc","namespace":"test-ns-whn67"},"namespace":"test-ns-whn67","name":"test-runtimepatches-trainjob-w2wqc","resource":{"group":"trainer.kubeflow.org","version":"v1alpha1","resource":"trainjobs"},"user":"system:serviceaccount:openshift-kueue-operator:kueue-controller-manager","requestID":"afc4d80d-4d43-465f-b6a6-a25b09aa9fac","replica-role":"follower"}