go test ./test/e2e/ocp/ -v -timeout 30m Starting OCP e2e test suite === RUN TestTrainJobWithClusterTrainingRuntime trainjob_test.go:43: Failed to create Trainer CR Unexpected error: <*errors.StatusError | 0x3d00df26b720>: object is being deleted: trainers.components.platform.opendatahub.io "default-trainer" already exists { ErrStatus: { TypeMeta: {Kind: "", APIVersion: ""}, ListMeta: { SelfLink: "", ResourceVersion: "", Continue: "", RemainingItemCount: nil, }, Status: "Failure", Message: "object is being deleted: trainers.components.platform.opendatahub.io \"default-trainer\" already exists", Reason: "AlreadyExists", Details: { Name: "default-trainer", Group: "components.platform.opendatahub.io", Kind: "trainers", UID: "", Causes: nil, RetryAfterSeconds: 0, }, Code: 409, }, } occurred client.go:139: Controller logs: 2026-07-27T17:52:29Z INFO setup starting manager 2026-07-27T17:52:29Z INFO controller-runtime.metrics Starting metrics server 2026-07-27T17:52:29Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": ":8080", "secure": false} 2026-07-27T17:52:29Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2026-07-27T17:52:29Z INFO Starting EventSource {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "source": "kind source: *unstructured.Unstructured"} 2026-07-27T17:52:29Z INFO Starting EventSource {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "source": "kind source: *unstructured.Unstructured"} 2026-07-27T17:52:29Z INFO Starting EventSource {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "source": "kind source: *unstructured.Unstructured"} 2026-07-27T17:52:29Z INFO Starting EventSource {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "source": "kind source: *unstructured.Unstructured"} 2026-07-27T17:52:29Z INFO Starting EventSource {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "source": "kind source: *unstructured.Unstructured"} 2026-07-27T17:52:30Z INFO Starting Controller {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer"} 2026-07-27T17:52:30Z INFO Starting workers {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "worker count": 1} 2026-07-27T17:53:18Z INFO reconcile {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "0693a01c-63e2-48bf-9344-92155c64942b"} 2026-07-27T17:53:18Z INFO adding finalizer {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "0693a01c-63e2-48bf-9344-92155c64942b"} 2026-07-27T17:53:18Z INFO apply {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "0693a01c-63e2-48bf-9344-92155c64942b"} 2026-07-27T17:53:18Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "0693a01c-63e2-48bf-9344-92155c64942b", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).checkDependencies-fm"} 2026-07-27T17:53:19Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "0693a01c-63e2-48bf-9344-92155c64942b", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).ensureNamespace-fm"} 2026-07-27T17:53:19Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).ensureNamespace-fm Created namespace {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "0693a01c-63e2-48bf-9344-92155c64942b", "namespace": "opendatahub"} 2026-07-27T17:53:19Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "0693a01c-63e2-48bf-9344-92155c64942b", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).updateReleases-fm"} 2026-07-27T17:53:19Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "0693a01c-63e2-48bf-9344-92155c64942b", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm"} 2026/07/27 17:53:20 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-07-27T17:53:20Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered manifests {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "0693a01c-63e2-48bf-9344-92155c64942b", "count": 34} 2026/07/27 17:53:20 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-07-27T17:53:20Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered runtimes {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "0693a01c-63e2-48bf-9344-92155c64942b", "count": 15} 2026/07/27 17:53:20 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-07-27T17:53:20Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered imagestreams {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "0693a01c-63e2-48bf-9344-92155c64942b", "count": 3} 2026-07-27T17:53:20Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "0693a01c-63e2-48bf-9344-92155c64942b", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.NewReconciler.withImmutableFieldRecovery.func9"} 2026-07-27T17:53:21Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "0693a01c-63e2-48bf-9344-92155c64942b", "action": "github.com/opendatahub-io/odh-platform-utilities/framework/controller/actions/status/deployments.(*Action).run-fm"} 2026-07-27T17:53:21Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "0693a01c-63e2-48bf-9344-92155c64942b", "action": "github.com/opendatahub-io/odh-platform-utilities/framework/controller/actions/gc.(*Action).run-fm"} 2026-07-27T17:53:23Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "0693a01c-63e2-48bf-9344-92155c64942b", "action": "github.com/opendatahub-io/odh-platform-utilities/framework/controller/reconciler.(*dynamicWatchAction).run-fm"} 2026-07-27T17:53:23Z INFO Starting EventSource {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "source": "kind source: *unstructured.Unstructured"} 2026-07-27T17:53:23Z INFO reconcile {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "87829be5-d174-4134-8d76-0fa78472ddd0"} 2026-07-27T17:53:23Z INFO apply {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "87829be5-d174-4134-8d76-0fa78472ddd0"} 2026-07-27T17:53:23Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "87829be5-d174-4134-8d76-0fa78472ddd0", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).checkDependencies-fm"} 2026-07-27T17:53:24Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "87829be5-d174-4134-8d76-0fa78472ddd0", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).ensureNamespace-fm"} 2026-07-27T17:53:24Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "87829be5-d174-4134-8d76-0fa78472ddd0", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).updateReleases-fm"} 2026-07-27T17:53:24Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "87829be5-d174-4134-8d76-0fa78472ddd0", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm"} 2026/07/27 17:53:25 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-07-27T17:53:25Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered manifests {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "87829be5-d174-4134-8d76-0fa78472ddd0", "count": 34} 2026/07/27 17:53:25 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-07-27T17:53:25Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered runtimes {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "87829be5-d174-4134-8d76-0fa78472ddd0", "count": 15} 2026/07/27 17:53:25 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-07-27T17:53:25Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered imagestreams {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "87829be5-d174-4134-8d76-0fa78472ddd0", "count": 3} 2026-07-27T17:53:25Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "87829be5-d174-4134-8d76-0fa78472ddd0", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.NewReconciler.withImmutableFieldRecovery.func9"} 2026-07-27T17:53:26Z ERROR Reconciler error {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "87829be5-d174-4134-8d76-0fa78472ddd0", "error": "provisioning failed: failure deploying resource /torch-distributed: apply failed trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime: unable to patch trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime torch-distributed: Internal error occurred: failed calling webhook \"validator.clustertrainingruntime.trainer.kubeflow.org\": failed to call webhook: Post \"https://kubeflow-trainer-controller-manager.opendatahub.svc:443/validate-trainer-kubeflow-org-v1alpha1-clustertrainingruntime?timeout=10s\": no endpoints available for service \"kubeflow-trainer-controller-manager\""} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313 2026-07-27T17:53:26Z INFO reconcile {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "efe4ebda-a4f6-4f22-82cc-4ab15f01e8db"} 2026-07-27T17:53:26Z INFO apply {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "efe4ebda-a4f6-4f22-82cc-4ab15f01e8db"} 2026-07-27T17:53:26Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "efe4ebda-a4f6-4f22-82cc-4ab15f01e8db", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).checkDependencies-fm"} 2026-07-27T17:53:26Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "efe4ebda-a4f6-4f22-82cc-4ab15f01e8db", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).ensureNamespace-fm"} 2026-07-27T17:53:26Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "efe4ebda-a4f6-4f22-82cc-4ab15f01e8db", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).updateReleases-fm"} 2026-07-27T17:53:26Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "efe4ebda-a4f6-4f22-82cc-4ab15f01e8db", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm"} 2026/07/27 17:53:27 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-07-27T17:53:27Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered manifests {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "efe4ebda-a4f6-4f22-82cc-4ab15f01e8db", "count": 34} 2026/07/27 17:53:27 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-07-27T17:53:27Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered runtimes {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "efe4ebda-a4f6-4f22-82cc-4ab15f01e8db", "count": 15} 2026/07/27 17:53:27 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-07-27T17:53:27Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered imagestreams {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "efe4ebda-a4f6-4f22-82cc-4ab15f01e8db", "count": 3} 2026-07-27T17:53:27Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "efe4ebda-a4f6-4f22-82cc-4ab15f01e8db", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.NewReconciler.withImmutableFieldRecovery.func9"} 2026-07-27T17:53:27Z ERROR Reconciler error {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "efe4ebda-a4f6-4f22-82cc-4ab15f01e8db", "error": "provisioning failed: failure deploying resource /torch-distributed: apply failed trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime: unable to patch trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime torch-distributed: Internal error occurred: failed calling webhook \"validator.clustertrainingruntime.trainer.kubeflow.org\": failed to call webhook: Post \"https://kubeflow-trainer-controller-manager.opendatahub.svc:443/validate-trainer-kubeflow-org-v1alpha1-clustertrainingruntime?timeout=10s\": no endpoints available for service \"kubeflow-trainer-controller-manager\""} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313 2026-07-27T17:53:27Z INFO reconcile {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "18e986a4-fd55-46ee-bba6-eaa5191d82da"} 2026-07-27T17:53:27Z INFO apply {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "18e986a4-fd55-46ee-bba6-eaa5191d82da"} 2026-07-27T17:53:27Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "18e986a4-fd55-46ee-bba6-eaa5191d82da", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).checkDependencies-fm"} 2026-07-27T17:53:27Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "18e986a4-fd55-46ee-bba6-eaa5191d82da", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).ensureNamespace-fm"} 2026-07-27T17:53:27Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "18e986a4-fd55-46ee-bba6-eaa5191d82da", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).updateReleases-fm"} 2026-07-27T17:53:27Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "18e986a4-fd55-46ee-bba6-eaa5191d82da", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm"} 2026/07/27 17:53:28 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-07-27T17:53:28Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered manifests {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "18e986a4-fd55-46ee-bba6-eaa5191d82da", "count": 34} 2026/07/27 17:53:28 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-07-27T17:53:28Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered runtimes {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "18e986a4-fd55-46ee-bba6-eaa5191d82da", "count": 15} 2026/07/27 17:53:28 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-07-27T17:53:28Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered imagestreams {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "18e986a4-fd55-46ee-bba6-eaa5191d82da", "count": 3} 2026-07-27T17:53:28Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "18e986a4-fd55-46ee-bba6-eaa5191d82da", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.NewReconciler.withImmutableFieldRecovery.func9"} 2026-07-27T17:53:28Z ERROR Reconciler error {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "18e986a4-fd55-46ee-bba6-eaa5191d82da", "error": "provisioning failed: failure deploying resource /torch-distributed: apply failed trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime: unable to patch trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime torch-distributed: Internal error occurred: failed calling webhook \"validator.clustertrainingruntime.trainer.kubeflow.org\": failed to call webhook: Post \"https://kubeflow-trainer-controller-manager.opendatahub.svc:443/validate-trainer-kubeflow-org-v1alpha1-clustertrainingruntime?timeout=10s\": no endpoints available for service \"kubeflow-trainer-controller-manager\""} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313 2026-07-27T17:53:28Z INFO reconcile {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "8fbc543f-cf2f-476b-b9d7-eff2ef377838"} 2026-07-27T17:53:28Z INFO apply {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "8fbc543f-cf2f-476b-b9d7-eff2ef377838"} 2026-07-27T17:53:28Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "8fbc543f-cf2f-476b-b9d7-eff2ef377838", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).checkDependencies-fm"} 2026-07-27T17:53:28Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "8fbc543f-cf2f-476b-b9d7-eff2ef377838", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).ensureNamespace-fm"} 2026-07-27T17:53:28Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "8fbc543f-cf2f-476b-b9d7-eff2ef377838", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).updateReleases-fm"} 2026-07-27T17:53:28Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "8fbc543f-cf2f-476b-b9d7-eff2ef377838", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm"} 2026/07/27 17:53:29 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-07-27T17:53:29Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered manifests {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "8fbc543f-cf2f-476b-b9d7-eff2ef377838", "count": 34} 2026/07/27 17:53:29 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-07-27T17:53:29Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered runtimes {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "8fbc543f-cf2f-476b-b9d7-eff2ef377838", "count": 15} 2026/07/27 17:53:29 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-07-27T17:53:29Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered imagestreams {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "8fbc543f-cf2f-476b-b9d7-eff2ef377838", "count": 3} 2026-07-27T17:53:30Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "8fbc543f-cf2f-476b-b9d7-eff2ef377838", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.NewReconciler.withImmutableFieldRecovery.func9"} 2026-07-27T17:53:30Z ERROR Reconciler error {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "8fbc543f-cf2f-476b-b9d7-eff2ef377838", "error": "provisioning failed: failure deploying resource /torch-distributed: apply failed trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime: unable to patch trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime torch-distributed: Internal error occurred: failed calling webhook \"validator.clustertrainingruntime.trainer.kubeflow.org\": failed to call webhook: Post \"https://kubeflow-trainer-controller-manager.opendatahub.svc:443/validate-trainer-kubeflow-org-v1alpha1-clustertrainingruntime?timeout=10s\": no endpoints available for service \"kubeflow-trainer-controller-manager\""} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313 2026-07-27T17:53:30Z INFO reconcile {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "eb4dbe50-7a67-4f0a-853b-3866df432b3f"} 2026-07-27T17:53:30Z INFO apply {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "eb4dbe50-7a67-4f0a-853b-3866df432b3f"} 2026-07-27T17:53:30Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "eb4dbe50-7a67-4f0a-853b-3866df432b3f", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).checkDependencies-fm"} 2026-07-27T17:53:30Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "eb4dbe50-7a67-4f0a-853b-3866df432b3f", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).ensureNamespace-fm"} 2026-07-27T17:53:30Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "eb4dbe50-7a67-4f0a-853b-3866df432b3f", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).updateReleases-fm"} 2026-07-27T17:53:30Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "eb4dbe50-7a67-4f0a-853b-3866df432b3f", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm"} 2026/07/27 17:53:31 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-07-27T17:53:31Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered manifests {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "eb4dbe50-7a67-4f0a-853b-3866df432b3f", "count": 34} 2026/07/27 17:53:31 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-07-27T17:53:31Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered runtimes {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "eb4dbe50-7a67-4f0a-853b-3866df432b3f", "count": 15} 2026/07/27 17:53:31 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-07-27T17:53:31Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered imagestreams {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "eb4dbe50-7a67-4f0a-853b-3866df432b3f", "count": 3} 2026-07-27T17:53:31Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "eb4dbe50-7a67-4f0a-853b-3866df432b3f", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.NewReconciler.withImmutableFieldRecovery.func9"} 2026-07-27T17:53:31Z ERROR Reconciler error {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "eb4dbe50-7a67-4f0a-853b-3866df432b3f", "error": "provisioning failed: failure deploying resource /torch-distributed: apply failed trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime: unable to patch trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime torch-distributed: Internal error occurred: failed calling webhook \"validator.clustertrainingruntime.trainer.kubeflow.org\": failed to call webhook: Post \"https://kubeflow-trainer-controller-manager.opendatahub.svc:443/validate-trainer-kubeflow-org-v1alpha1-clustertrainingruntime?timeout=10s\": no endpoints available for service \"kubeflow-trainer-controller-manager\""} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313 2026-07-27T17:53:31Z INFO reconcile {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "8422b533-1e3b-4252-a0b0-dbc10bd76a27"} 2026-07-27T17:53:31Z INFO apply {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "8422b533-1e3b-4252-a0b0-dbc10bd76a27"} 2026-07-27T17:53:31Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "8422b533-1e3b-4252-a0b0-dbc10bd76a27", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).checkDependencies-fm"} 2026-07-27T17:53:31Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "8422b533-1e3b-4252-a0b0-dbc10bd76a27", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).ensureNamespace-fm"} 2026-07-27T17:53:31Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "8422b533-1e3b-4252-a0b0-dbc10bd76a27", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).updateReleases-fm"} 2026-07-27T17:53:31Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "8422b533-1e3b-4252-a0b0-dbc10bd76a27", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm"} 2026/07/27 17:53:32 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-07-27T17:53:32Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered manifests {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "8422b533-1e3b-4252-a0b0-dbc10bd76a27", "count": 34} 2026/07/27 17:53:32 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-07-27T17:53:32Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered runtimes {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "8422b533-1e3b-4252-a0b0-dbc10bd76a27", "count": 15} 2026/07/27 17:53:32 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-07-27T17:53:32Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered imagestreams {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "8422b533-1e3b-4252-a0b0-dbc10bd76a27", "count": 3} 2026-07-27T17:53:32Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "8422b533-1e3b-4252-a0b0-dbc10bd76a27", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.NewReconciler.withImmutableFieldRecovery.func9"} 2026-07-27T17:53:33Z ERROR Reconciler error {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "8422b533-1e3b-4252-a0b0-dbc10bd76a27", "error": "provisioning failed: failure deploying resource /torch-distributed: apply failed trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime: unable to patch trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime torch-distributed: Internal error occurred: failed calling webhook \"validator.clustertrainingruntime.trainer.kubeflow.org\": failed to call webhook: Post \"https://kubeflow-trainer-controller-manager.opendatahub.svc:443/validate-trainer-kubeflow-org-v1alpha1-clustertrainingruntime?timeout=10s\": no endpoints available for service \"kubeflow-trainer-controller-manager\""} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313 2026-07-27T17:53:33Z INFO reconcile {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "05580769-1b1f-470a-be4d-642dd0b91dad"} 2026-07-27T17:53:33Z INFO apply {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "05580769-1b1f-470a-be4d-642dd0b91dad"} 2026-07-27T17:53:33Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "05580769-1b1f-470a-be4d-642dd0b91dad", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).checkDependencies-fm"} 2026-07-27T17:53:33Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "05580769-1b1f-470a-be4d-642dd0b91dad", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).ensureNamespace-fm"} 2026-07-27T17:53:33Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "05580769-1b1f-470a-be4d-642dd0b91dad", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).updateReleases-fm"} 2026-07-27T17:53:33Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "05580769-1b1f-470a-be4d-642dd0b91dad", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm"} 2026/07/27 17:53:34 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-07-27T17:53:34Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered manifests {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "05580769-1b1f-470a-be4d-642dd0b91dad", "count": 34} 2026/07/27 17:53:34 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-07-27T17:53:34Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered runtimes {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "05580769-1b1f-470a-be4d-642dd0b91dad", "count": 15} 2026/07/27 17:53:34 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-07-27T17:53:34Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered imagestreams {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "05580769-1b1f-470a-be4d-642dd0b91dad", "count": 3} 2026-07-27T17:53:34Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "05580769-1b1f-470a-be4d-642dd0b91dad", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.NewReconciler.withImmutableFieldRecovery.func9"} 2026-07-27T17:53:34Z ERROR Reconciler error {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "05580769-1b1f-470a-be4d-642dd0b91dad", "error": "provisioning failed: failure deploying resource /torch-distributed: apply failed trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime: unable to patch trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime torch-distributed: Internal error occurred: failed calling webhook \"validator.clustertrainingruntime.trainer.kubeflow.org\": failed to call webhook: Post \"https://kubeflow-trainer-controller-manager.opendatahub.svc:443/validate-trainer-kubeflow-org-v1alpha1-clustertrainingruntime?timeout=10s\": no endpoints available for service \"kubeflow-trainer-controller-manager\""} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313 2026-07-27T17:53:35Z INFO reconcile {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "fd2ee1ad-216d-453d-9dce-6113d7da867e"} 2026-07-27T17:53:35Z INFO apply {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "fd2ee1ad-216d-453d-9dce-6113d7da867e"} 2026-07-27T17:53:35Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "fd2ee1ad-216d-453d-9dce-6113d7da867e", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).checkDependencies-fm"} 2026-07-27T17:53:35Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "fd2ee1ad-216d-453d-9dce-6113d7da867e", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).ensureNamespace-fm"} 2026-07-27T17:53:35Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "fd2ee1ad-216d-453d-9dce-6113d7da867e", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).updateReleases-fm"} 2026-07-27T17:53:35Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "fd2ee1ad-216d-453d-9dce-6113d7da867e", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm"} 2026/07/27 17:53:36 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-07-27T17:53:36Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered manifests {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "fd2ee1ad-216d-453d-9dce-6113d7da867e", "count": 34} 2026/07/27 17:53:36 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-07-27T17:53:36Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered runtimes {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "fd2ee1ad-216d-453d-9dce-6113d7da867e", "count": 15} 2026/07/27 17:53:36 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-07-27T17:53:36Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered imagestreams {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "fd2ee1ad-216d-453d-9dce-6113d7da867e", "count": 3} 2026-07-27T17:53:36Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "fd2ee1ad-216d-453d-9dce-6113d7da867e", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.NewReconciler.withImmutableFieldRecovery.func9"} 2026-07-27T17:53:36Z ERROR Reconciler error {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "fd2ee1ad-216d-453d-9dce-6113d7da867e", "error": "provisioning failed: failure deploying resource /torch-distributed: apply failed trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime: unable to patch trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime torch-distributed: Internal error occurred: failed calling webhook \"validator.clustertrainingruntime.trainer.kubeflow.org\": failed to call webhook: Post \"https://kubeflow-trainer-controller-manager.opendatahub.svc:443/validate-trainer-kubeflow-org-v1alpha1-clustertrainingruntime?timeout=10s\": no endpoints available for service \"kubeflow-trainer-controller-manager\""} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313 2026-07-27T17:53:37Z INFO reconcile {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "7080f397-7d44-4209-aac9-22ff8e542ad0"} 2026-07-27T17:53:37Z INFO apply {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "7080f397-7d44-4209-aac9-22ff8e542ad0"} 2026-07-27T17:53:37Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "7080f397-7d44-4209-aac9-22ff8e542ad0", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).checkDependencies-fm"} 2026-07-27T17:53:37Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "7080f397-7d44-4209-aac9-22ff8e542ad0", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).ensureNamespace-fm"} 2026-07-27T17:53:37Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "7080f397-7d44-4209-aac9-22ff8e542ad0", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).updateReleases-fm"} 2026-07-27T17:53:37Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "7080f397-7d44-4209-aac9-22ff8e542ad0", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm"} 2026/07/27 17:53:38 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-07-27T17:53:38Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered manifests {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "7080f397-7d44-4209-aac9-22ff8e542ad0", "count": 34} 2026/07/27 17:53:38 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-07-27T17:53:38Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered runtimes {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "7080f397-7d44-4209-aac9-22ff8e542ad0", "count": 15} 2026/07/27 17:53:38 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-07-27T17:53:38Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered imagestreams {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "7080f397-7d44-4209-aac9-22ff8e542ad0", "count": 3} 2026-07-27T17:53:38Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "7080f397-7d44-4209-aac9-22ff8e542ad0", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.NewReconciler.withImmutableFieldRecovery.func9"} 2026-07-27T17:53:38Z ERROR Reconciler error {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "7080f397-7d44-4209-aac9-22ff8e542ad0", "error": "provisioning failed: failure deploying resource /torch-distributed: apply failed trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime: unable to patch trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime torch-distributed: Internal error occurred: failed calling webhook \"validator.clustertrainingruntime.trainer.kubeflow.org\": failed to call webhook: Post \"https://kubeflow-trainer-controller-manager.opendatahub.svc:443/validate-trainer-kubeflow-org-v1alpha1-clustertrainingruntime?timeout=10s\": no endpoints available for service \"kubeflow-trainer-controller-manager\""} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313 2026-07-27T17:53:40Z INFO reconcile {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "00ed0f15-0c11-4752-82f3-7b2cde994a06"} 2026-07-27T17:53:40Z INFO apply {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "00ed0f15-0c11-4752-82f3-7b2cde994a06"} 2026-07-27T17:53:40Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "00ed0f15-0c11-4752-82f3-7b2cde994a06", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).checkDependencies-fm"} 2026-07-27T17:53:40Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "00ed0f15-0c11-4752-82f3-7b2cde994a06", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).ensureNamespace-fm"} 2026-07-27T17:53:40Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "00ed0f15-0c11-4752-82f3-7b2cde994a06", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).updateReleases-fm"} 2026-07-27T17:53:40Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "00ed0f15-0c11-4752-82f3-7b2cde994a06", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm"} 2026/07/27 17:53:41 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-07-27T17:53:41Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered manifests {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "00ed0f15-0c11-4752-82f3-7b2cde994a06", "count": 34} 2026/07/27 17:53:41 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-07-27T17:53:41Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered runtimes {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "00ed0f15-0c11-4752-82f3-7b2cde994a06", "count": 15} 2026/07/27 17:53:41 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-07-27T17:53:41Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered imagestreams {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "00ed0f15-0c11-4752-82f3-7b2cde994a06", "count": 3} 2026-07-27T17:53:41Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "00ed0f15-0c11-4752-82f3-7b2cde994a06", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.NewReconciler.withImmutableFieldRecovery.func9"} 2026-07-27T17:53:41Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "00ed0f15-0c11-4752-82f3-7b2cde994a06", "action": "github.com/opendatahub-io/odh-platform-utilities/framework/controller/actions/status/deployments.(*Action).run-fm"} 2026-07-27T17:53:41Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "00ed0f15-0c11-4752-82f3-7b2cde994a06", "action": "github.com/opendatahub-io/odh-platform-utilities/framework/controller/actions/gc.(*Action).run-fm"} 2026-07-27T17:53:41Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "00ed0f15-0c11-4752-82f3-7b2cde994a06", "action": "github.com/opendatahub-io/odh-platform-utilities/framework/controller/reconciler.(*dynamicWatchAction).run-fm"} 2026-07-27T17:54:57Z INFO reconcile {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "e88b4c14-3535-4c3c-a1be-12a96ef521e6"} 2026-07-27T17:54:57Z INFO apply {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "e88b4c14-3535-4c3c-a1be-12a96ef521e6"} 2026-07-27T17:54:57Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "e88b4c14-3535-4c3c-a1be-12a96ef521e6", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).checkDependencies-fm"} 2026-07-27T17:54:57Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "e88b4c14-3535-4c3c-a1be-12a96ef521e6", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).ensureNamespace-fm"} 2026-07-27T17:54:57Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "e88b4c14-3535-4c3c-a1be-12a96ef521e6", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).updateReleases-fm"} 2026-07-27T17:54:57Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "e88b4c14-3535-4c3c-a1be-12a96ef521e6", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm"} 2026/07/27 17:54:58 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-07-27T17:54:58Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered manifests {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "e88b4c14-3535-4c3c-a1be-12a96ef521e6", "count": 34} 2026/07/27 17:54:58 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-07-27T17:54:58Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered runtimes {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "e88b4c14-3535-4c3c-a1be-12a96ef521e6", "count": 15} 2026/07/27 17:54:58 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-07-27T17:54:58Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered imagestreams {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "e88b4c14-3535-4c3c-a1be-12a96ef521e6", "count": 3} 2026-07-27T17:54:58Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "e88b4c14-3535-4c3c-a1be-12a96ef521e6", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.NewReconciler.withImmutableFieldRecovery.func9"} 2026-07-27T17:54:59Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "e88b4c14-3535-4c3c-a1be-12a96ef521e6", "action": "github.com/opendatahub-io/odh-platform-utilities/framework/controller/actions/status/deployments.(*Action).run-fm"} 2026-07-27T17:54:59Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "e88b4c14-3535-4c3c-a1be-12a96ef521e6", "action": "github.com/opendatahub-io/odh-platform-utilities/framework/controller/actions/gc.(*Action).run-fm"} 2026-07-27T17:54:59Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "e88b4c14-3535-4c3c-a1be-12a96ef521e6", "action": "github.com/opendatahub-io/odh-platform-utilities/framework/controller/reconciler.(*dynamicWatchAction).run-fm"} client.go:156: Kubernetes events: 0001-01-01T00:00:00Z Normal Scheduled trainer-operator-controller-manager-56f5b7fd56-gdmqt Successfully assigned trainer-operator-system/trainer-operator-controller-manager-56f5b7fd56-gdmqt to ip-10-0-128-74.ec2.internal 0001-01-01T00:00:00Z Normal Scheduled trainer-operator-controller-manager-7b9d4cb9c4-5vzp5 Successfully assigned trainer-operator-system/trainer-operator-controller-manager-7b9d4cb9c4-5vzp5 to ip-10-0-133-31.ec2.internal 2026-07-27T17:52:26Z Normal SuccessfulCreate trainer-operator-controller-manager-56f5b7fd56 Created pod: trainer-operator-controller-manager-56f5b7fd56-gdmqt 2026-07-27T17:52:26Z Normal AddedInterface trainer-operator-controller-manager-56f5b7fd56-gdmqt Add eth0 [10.132.0.22/23] from ovn-kubernetes 2026-07-27T17:52:26Z Normal Pulling trainer-operator-controller-manager-56f5b7fd56-gdmqt Pulling image "quay.io/opendatahub/odh-trainer-operator:odh-stable" 2026-07-27T17:52:26Z Normal ScalingReplicaSet trainer-operator-controller-manager Scaled up replica set trainer-operator-controller-manager-56f5b7fd56 from 0 to 1 2026-07-27T17:52:29Z Normal Pulled trainer-operator-controller-manager-56f5b7fd56-gdmqt Successfully pulled image "quay.io/opendatahub/odh-trainer-operator:odh-stable" in 2.56s (2.56s including waiting). Image size: 197977280 bytes. 2026-07-27T17:52:29Z Normal Started trainer-operator-controller-manager-56f5b7fd56-gdmqt Started container manager 2026-07-27T17:52:29Z Normal Created trainer-operator-controller-manager-56f5b7fd56-gdmqt Created container: manager 2026-07-27T17:55:00Z Normal SuccessfulCreate trainer-operator-controller-manager-7b9d4cb9c4 Created pod: trainer-operator-controller-manager-7b9d4cb9c4-5vzp5 2026-07-27T17:55:00Z Normal ScalingReplicaSet trainer-operator-controller-manager Scaled up replica set trainer-operator-controller-manager-7b9d4cb9c4 from 0 to 1 2026-07-27T17:55:01Z Normal AddedInterface trainer-operator-controller-manager-7b9d4cb9c4-5vzp5 Add eth0 [10.133.0.23/23] from ovn-kubernetes 2026-07-27T17:55:01Z Normal Pulling trainer-operator-controller-manager-7b9d4cb9c4-5vzp5 Pulling image "quay.io/opendatahub/odh-trainer-operator@sha256:4a9bfea3194fff7393d94e351494ff09561eb07e46588b197fa4ebe6dd1fda9a" --- FAIL: TestTrainJobWithClusterTrainingRuntime (3.16s) FAIL FAIL github.com/opendatahub-io/trainer-operator/test/e2e/ocp 3.205s FAIL make: *** [Makefile:151: test-e2e-ocp] Error 1