go test ./test/e2e/ocp/ -v -timeout 30m Starting OCP e2e test suite === RUN TestTrainJobWithClusterTrainingRuntime trainjob_test.go:53: Timed out after 120.001s. The function passed to Eventually failed at /workspace/source/src_code/test/e2e/ocp/trainjob_test.go:52 with: Expected : Not Ready to equal : Ready client.go:143: Controller logs: 2026-08-01T03:53:54Z INFO setup starting manager 2026-08-01T03:53:54Z INFO controller-runtime.metrics Starting metrics server 2026-08-01T03:53:54Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2026-08-01T03:53:54Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": ":8080", "secure": false} 2026-08-01T03:53:54Z INFO Starting EventSource {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "source": "kind source: *unstructured.Unstructured"} 2026-08-01T03:53:54Z INFO Starting EventSource {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "source": "kind source: *unstructured.Unstructured"} 2026-08-01T03:53:54Z INFO Starting EventSource {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "source": "kind source: *unstructured.Unstructured"} 2026-08-01T03:53:54Z INFO Starting EventSource {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "source": "kind source: *unstructured.Unstructured"} 2026-08-01T03:53:54Z INFO Starting EventSource {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "source": "kind source: *unstructured.Unstructured"} 2026-08-01T03:53:55Z INFO Starting Controller {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer"} 2026-08-01T03:53:55Z INFO Starting workers {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "worker count": 1} 2026-08-01T03:54:40Z INFO reconcile {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "ce5166d4-8fd8-4df8-83ab-d1450868debe"} 2026-08-01T03:54:40Z INFO adding finalizer {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "ce5166d4-8fd8-4df8-83ab-d1450868debe"} 2026-08-01T03:54:40Z INFO apply {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "ce5166d4-8fd8-4df8-83ab-d1450868debe"} 2026-08-01T03:54:40Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "ce5166d4-8fd8-4df8-83ab-d1450868debe", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).checkDependencies-fm"} 2026-08-01T03:54:40Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "ce5166d4-8fd8-4df8-83ab-d1450868debe", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).ensureNamespace-fm"} 2026-08-01T03:54:40Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).ensureNamespace-fm Created namespace {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "ce5166d4-8fd8-4df8-83ab-d1450868debe", "namespace": "opendatahub"} 2026-08-01T03:54:40Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "ce5166d4-8fd8-4df8-83ab-d1450868debe", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).updateReleases-fm"} 2026-08-01T03:54:40Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "ce5166d4-8fd8-4df8-83ab-d1450868debe", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm"} 2026/08/01 03:54:41 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T03:54:41Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered manifests {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "ce5166d4-8fd8-4df8-83ab-d1450868debe", "count": 31} 2026/08/01 03:54:42 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T03:54:42Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered runtimes {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "ce5166d4-8fd8-4df8-83ab-d1450868debe", "count": 15} 2026/08/01 03:54:42 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T03:54:42Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered imagestreams {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "ce5166d4-8fd8-4df8-83ab-d1450868debe", "count": 3} 2026-08-01T03:54:42Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "ce5166d4-8fd8-4df8-83ab-d1450868debe", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.NewReconciler.withImmutableFieldRecovery.func9"} 2026-08-01T03:54:43Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "ce5166d4-8fd8-4df8-83ab-d1450868debe", "action": "github.com/opendatahub-io/odh-platform-utilities/framework/controller/actions/status/deployments.(*Action).run-fm"} 2026-08-01T03:54:43Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "ce5166d4-8fd8-4df8-83ab-d1450868debe", "action": "github.com/opendatahub-io/odh-platform-utilities/framework/controller/actions/gc.(*Action).run-fm"} 2026-08-01T03:54:44Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "ce5166d4-8fd8-4df8-83ab-d1450868debe", "action": "github.com/opendatahub-io/odh-platform-utilities/framework/controller/reconciler.(*dynamicWatchAction).run-fm"} 2026-08-01T03:54:45Z INFO Starting EventSource {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "source": "kind source: *unstructured.Unstructured"} 2026-08-01T03:54:45Z INFO reconcile {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "6443669e-690d-414c-9406-6e716f26f1d4"} 2026-08-01T03:54:45Z INFO apply {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "6443669e-690d-414c-9406-6e716f26f1d4"} 2026-08-01T03:54:45Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "6443669e-690d-414c-9406-6e716f26f1d4", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).checkDependencies-fm"} 2026-08-01T03:54:45Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "6443669e-690d-414c-9406-6e716f26f1d4", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).ensureNamespace-fm"} 2026-08-01T03:54:45Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "6443669e-690d-414c-9406-6e716f26f1d4", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).updateReleases-fm"} 2026-08-01T03:54:45Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "6443669e-690d-414c-9406-6e716f26f1d4", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm"} 2026/08/01 03:54:46 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T03:54:46Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered manifests {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "6443669e-690d-414c-9406-6e716f26f1d4", "count": 31} 2026/08/01 03:54:46 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T03:54:46Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered runtimes {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "6443669e-690d-414c-9406-6e716f26f1d4", "count": 15} 2026/08/01 03:54:46 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T03:54:46Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered imagestreams {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "6443669e-690d-414c-9406-6e716f26f1d4", "count": 3} 2026-08-01T03:54:46Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "6443669e-690d-414c-9406-6e716f26f1d4", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.NewReconciler.withImmutableFieldRecovery.func9"} 2026-08-01T03:54:47Z ERROR Reconciler error {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "6443669e-690d-414c-9406-6e716f26f1d4", "error": "provisioning failed: failure deploying resource /torch-distributed: apply failed trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime: unable to patch trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime torch-distributed: Internal error occurred: failed calling webhook \"validator.clustertrainingruntime.trainer.kubeflow.org\": failed to call webhook: Post \"https://kubeflow-trainer-controller-manager.opendatahub.svc:443/validate-trainer-kubeflow-org-v1alpha1-clustertrainingruntime?timeout=10s\": no endpoints available for service \"kubeflow-trainer-controller-manager\""} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313 2026-08-01T03:54:47Z INFO reconcile {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "7ef03c08-2141-4acd-9832-88a6873d73b6"} 2026-08-01T03:54:47Z INFO apply {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "7ef03c08-2141-4acd-9832-88a6873d73b6"} 2026-08-01T03:54:47Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "7ef03c08-2141-4acd-9832-88a6873d73b6", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).checkDependencies-fm"} 2026-08-01T03:54:47Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "7ef03c08-2141-4acd-9832-88a6873d73b6", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).ensureNamespace-fm"} 2026-08-01T03:54:47Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "7ef03c08-2141-4acd-9832-88a6873d73b6", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).updateReleases-fm"} 2026-08-01T03:54:47Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "7ef03c08-2141-4acd-9832-88a6873d73b6", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm"} 2026/08/01 03:54:48 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T03:54:48Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered manifests {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "7ef03c08-2141-4acd-9832-88a6873d73b6", "count": 31} 2026/08/01 03:54:48 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T03:54:48Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered runtimes {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "7ef03c08-2141-4acd-9832-88a6873d73b6", "count": 15} 2026/08/01 03:54:48 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T03:54:48Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered imagestreams {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "7ef03c08-2141-4acd-9832-88a6873d73b6", "count": 3} 2026-08-01T03:54:48Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "7ef03c08-2141-4acd-9832-88a6873d73b6", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.NewReconciler.withImmutableFieldRecovery.func9"} 2026-08-01T03:54:49Z ERROR Reconciler error {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "7ef03c08-2141-4acd-9832-88a6873d73b6", "error": "provisioning failed: failure deploying resource /torch-distributed: apply failed trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime: unable to patch trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime torch-distributed: Internal error occurred: failed calling webhook \"validator.clustertrainingruntime.trainer.kubeflow.org\": failed to call webhook: Post \"https://kubeflow-trainer-controller-manager.opendatahub.svc:443/validate-trainer-kubeflow-org-v1alpha1-clustertrainingruntime?timeout=10s\": no endpoints available for service \"kubeflow-trainer-controller-manager\""} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313 2026-08-01T03:54:49Z INFO reconcile {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "d7c49743-098d-41a7-8cb6-45e7fe5ba8c5"} 2026-08-01T03:54:49Z INFO apply {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "d7c49743-098d-41a7-8cb6-45e7fe5ba8c5"} 2026-08-01T03:54:49Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "d7c49743-098d-41a7-8cb6-45e7fe5ba8c5", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).checkDependencies-fm"} 2026-08-01T03:54:49Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "d7c49743-098d-41a7-8cb6-45e7fe5ba8c5", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).ensureNamespace-fm"} 2026-08-01T03:54:49Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "d7c49743-098d-41a7-8cb6-45e7fe5ba8c5", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).updateReleases-fm"} 2026-08-01T03:54:49Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "d7c49743-098d-41a7-8cb6-45e7fe5ba8c5", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm"} 2026/08/01 03:54:50 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T03:54:50Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered manifests {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "d7c49743-098d-41a7-8cb6-45e7fe5ba8c5", "count": 31} 2026/08/01 03:54:50 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T03:54:50Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered runtimes {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "d7c49743-098d-41a7-8cb6-45e7fe5ba8c5", "count": 15} 2026/08/01 03:54:50 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T03:54:50Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered imagestreams {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "d7c49743-098d-41a7-8cb6-45e7fe5ba8c5", "count": 3} 2026-08-01T03:54:50Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "d7c49743-098d-41a7-8cb6-45e7fe5ba8c5", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.NewReconciler.withImmutableFieldRecovery.func9"} 2026-08-01T03:54:50Z ERROR Reconciler error {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "d7c49743-098d-41a7-8cb6-45e7fe5ba8c5", "error": "provisioning failed: failure deploying resource /torch-distributed: apply failed trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime: unable to patch trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime torch-distributed: Internal error occurred: failed calling webhook \"validator.clustertrainingruntime.trainer.kubeflow.org\": failed to call webhook: Post \"https://kubeflow-trainer-controller-manager.opendatahub.svc:443/validate-trainer-kubeflow-org-v1alpha1-clustertrainingruntime?timeout=10s\": no endpoints available for service \"kubeflow-trainer-controller-manager\""} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313 2026-08-01T03:54:50Z INFO reconcile {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "a115c166-badd-411f-8f2c-c69da8d1bfe7"} 2026-08-01T03:54:50Z INFO apply {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "a115c166-badd-411f-8f2c-c69da8d1bfe7"} 2026-08-01T03:54:50Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "a115c166-badd-411f-8f2c-c69da8d1bfe7", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).checkDependencies-fm"} 2026-08-01T03:54:50Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "a115c166-badd-411f-8f2c-c69da8d1bfe7", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).ensureNamespace-fm"} 2026-08-01T03:54:50Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "a115c166-badd-411f-8f2c-c69da8d1bfe7", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).updateReleases-fm"} 2026-08-01T03:54:50Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "a115c166-badd-411f-8f2c-c69da8d1bfe7", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm"} 2026/08/01 03:54:51 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T03:54:51Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered manifests {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "a115c166-badd-411f-8f2c-c69da8d1bfe7", "count": 31} 2026/08/01 03:54:51 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T03:54:51Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered runtimes {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "a115c166-badd-411f-8f2c-c69da8d1bfe7", "count": 15} 2026/08/01 03:54:51 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T03:54:51Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered imagestreams {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "a115c166-badd-411f-8f2c-c69da8d1bfe7", "count": 3} 2026-08-01T03:54:51Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "a115c166-badd-411f-8f2c-c69da8d1bfe7", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.NewReconciler.withImmutableFieldRecovery.func9"} 2026-08-01T03:54:51Z ERROR Reconciler error {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "a115c166-badd-411f-8f2c-c69da8d1bfe7", "error": "provisioning failed: failure deploying resource /torch-distributed: apply failed trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime: unable to patch trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime torch-distributed: Internal error occurred: failed calling webhook \"validator.clustertrainingruntime.trainer.kubeflow.org\": failed to call webhook: Post \"https://kubeflow-trainer-controller-manager.opendatahub.svc:443/validate-trainer-kubeflow-org-v1alpha1-clustertrainingruntime?timeout=10s\": no endpoints available for service \"kubeflow-trainer-controller-manager\""} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313 2026-08-01T03:54:51Z INFO reconcile {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "f85dba3a-6869-4ab9-bd36-382198efda3d"} 2026-08-01T03:54:51Z INFO apply {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "f85dba3a-6869-4ab9-bd36-382198efda3d"} 2026-08-01T03:54:51Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "f85dba3a-6869-4ab9-bd36-382198efda3d", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).checkDependencies-fm"} 2026-08-01T03:54:52Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "f85dba3a-6869-4ab9-bd36-382198efda3d", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).ensureNamespace-fm"} 2026-08-01T03:54:52Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "f85dba3a-6869-4ab9-bd36-382198efda3d", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).updateReleases-fm"} 2026-08-01T03:54:52Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "f85dba3a-6869-4ab9-bd36-382198efda3d", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm"} 2026/08/01 03:54:53 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T03:54:53Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered manifests {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "f85dba3a-6869-4ab9-bd36-382198efda3d", "count": 31} 2026/08/01 03:54:53 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T03:54:53Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered runtimes {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "f85dba3a-6869-4ab9-bd36-382198efda3d", "count": 15} 2026/08/01 03:54:53 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T03:54:53Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered imagestreams {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "f85dba3a-6869-4ab9-bd36-382198efda3d", "count": 3} 2026-08-01T03:54:53Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "f85dba3a-6869-4ab9-bd36-382198efda3d", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.NewReconciler.withImmutableFieldRecovery.func9"} 2026-08-01T03:54:53Z ERROR Reconciler error {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "f85dba3a-6869-4ab9-bd36-382198efda3d", "error": "provisioning failed: failure deploying resource /torch-distributed: apply failed trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime: unable to patch trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime torch-distributed: Internal error occurred: failed calling webhook \"validator.clustertrainingruntime.trainer.kubeflow.org\": failed to call webhook: Post \"https://kubeflow-trainer-controller-manager.opendatahub.svc:443/validate-trainer-kubeflow-org-v1alpha1-clustertrainingruntime?timeout=10s\": no endpoints available for service \"kubeflow-trainer-controller-manager\""} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313 2026-08-01T03:54:53Z INFO reconcile {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "c3511789-6542-41c6-8735-2ac0e754727a"} 2026-08-01T03:54:53Z INFO apply {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "c3511789-6542-41c6-8735-2ac0e754727a"} 2026-08-01T03:54:53Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "c3511789-6542-41c6-8735-2ac0e754727a", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).checkDependencies-fm"} 2026-08-01T03:54:53Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "c3511789-6542-41c6-8735-2ac0e754727a", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).ensureNamespace-fm"} 2026-08-01T03:54:53Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "c3511789-6542-41c6-8735-2ac0e754727a", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).updateReleases-fm"} 2026-08-01T03:54:53Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "c3511789-6542-41c6-8735-2ac0e754727a", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm"} 2026/08/01 03:54:54 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T03:54:54Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered manifests {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "c3511789-6542-41c6-8735-2ac0e754727a", "count": 31} 2026/08/01 03:54:54 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T03:54:54Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered runtimes {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "c3511789-6542-41c6-8735-2ac0e754727a", "count": 15} 2026/08/01 03:54:54 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T03:54:54Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered imagestreams {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "c3511789-6542-41c6-8735-2ac0e754727a", "count": 3} 2026-08-01T03:54:54Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "c3511789-6542-41c6-8735-2ac0e754727a", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.NewReconciler.withImmutableFieldRecovery.func9"} 2026-08-01T03:54:54Z ERROR Reconciler error {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "c3511789-6542-41c6-8735-2ac0e754727a", "error": "provisioning failed: failure deploying resource /torch-distributed: apply failed trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime: unable to patch trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime torch-distributed: Internal error occurred: failed calling webhook \"validator.clustertrainingruntime.trainer.kubeflow.org\": failed to call webhook: Post \"https://kubeflow-trainer-controller-manager.opendatahub.svc:443/validate-trainer-kubeflow-org-v1alpha1-clustertrainingruntime?timeout=10s\": no endpoints available for service \"kubeflow-trainer-controller-manager\""} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313 2026-08-01T03:54:55Z INFO reconcile {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "b04fe247-f76a-4a74-a0a3-38ed670672f1"} 2026-08-01T03:54:55Z INFO apply {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "b04fe247-f76a-4a74-a0a3-38ed670672f1"} 2026-08-01T03:54:55Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "b04fe247-f76a-4a74-a0a3-38ed670672f1", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).checkDependencies-fm"} 2026-08-01T03:54:55Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "b04fe247-f76a-4a74-a0a3-38ed670672f1", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).ensureNamespace-fm"} 2026-08-01T03:54:55Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "b04fe247-f76a-4a74-a0a3-38ed670672f1", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).updateReleases-fm"} 2026-08-01T03:54:55Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "b04fe247-f76a-4a74-a0a3-38ed670672f1", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm"} 2026/08/01 03:54:56 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T03:54:56Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered manifests {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "b04fe247-f76a-4a74-a0a3-38ed670672f1", "count": 31} 2026/08/01 03:54:56 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T03:54:56Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered runtimes {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "b04fe247-f76a-4a74-a0a3-38ed670672f1", "count": 15} 2026/08/01 03:54:56 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T03:54:56Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered imagestreams {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "b04fe247-f76a-4a74-a0a3-38ed670672f1", "count": 3} 2026-08-01T03:54:56Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "b04fe247-f76a-4a74-a0a3-38ed670672f1", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.NewReconciler.withImmutableFieldRecovery.func9"} 2026-08-01T03:54:56Z ERROR Reconciler error {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "b04fe247-f76a-4a74-a0a3-38ed670672f1", "error": "provisioning failed: failure deploying resource /torch-distributed: apply failed trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime: unable to patch trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime torch-distributed: Internal error occurred: failed calling webhook \"validator.clustertrainingruntime.trainer.kubeflow.org\": failed to call webhook: Post \"https://kubeflow-trainer-controller-manager.opendatahub.svc:443/validate-trainer-kubeflow-org-v1alpha1-clustertrainingruntime?timeout=10s\": no endpoints available for service \"kubeflow-trainer-controller-manager\""} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313 2026-08-01T03:54:56Z INFO reconcile {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "eb26be3e-fefd-445f-ade4-a89dbd359532"} 2026-08-01T03:54:56Z INFO apply {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "eb26be3e-fefd-445f-ade4-a89dbd359532"} 2026-08-01T03:54:56Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "eb26be3e-fefd-445f-ade4-a89dbd359532", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).checkDependencies-fm"} 2026-08-01T03:54:56Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "eb26be3e-fefd-445f-ade4-a89dbd359532", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).ensureNamespace-fm"} 2026-08-01T03:54:56Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "eb26be3e-fefd-445f-ade4-a89dbd359532", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).updateReleases-fm"} 2026-08-01T03:54:56Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "eb26be3e-fefd-445f-ade4-a89dbd359532", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm"} 2026/08/01 03:54:57 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T03:54:57Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered manifests {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "eb26be3e-fefd-445f-ade4-a89dbd359532", "count": 31} 2026/08/01 03:54:57 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T03:54:57Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered runtimes {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "eb26be3e-fefd-445f-ade4-a89dbd359532", "count": 15} 2026/08/01 03:54:57 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T03:54:57Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered imagestreams {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "eb26be3e-fefd-445f-ade4-a89dbd359532", "count": 3} 2026-08-01T03:54:57Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "eb26be3e-fefd-445f-ade4-a89dbd359532", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.NewReconciler.withImmutableFieldRecovery.func9"} 2026-08-01T03:54:58Z ERROR Reconciler error {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "eb26be3e-fefd-445f-ade4-a89dbd359532", "error": "provisioning failed: failure deploying resource /torch-distributed: apply failed trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime: unable to patch trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime torch-distributed: Internal error occurred: failed calling webhook \"validator.clustertrainingruntime.trainer.kubeflow.org\": failed to call webhook: Post \"https://kubeflow-trainer-controller-manager.opendatahub.svc:443/validate-trainer-kubeflow-org-v1alpha1-clustertrainingruntime?timeout=10s\": no endpoints available for service \"kubeflow-trainer-controller-manager\""} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313 2026-08-01T03:54:58Z INFO reconcile {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "ec742bfe-6d10-4ef2-818a-887afa370635"} 2026-08-01T03:54:58Z INFO apply {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "ec742bfe-6d10-4ef2-818a-887afa370635"} 2026-08-01T03:54:58Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "ec742bfe-6d10-4ef2-818a-887afa370635", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).checkDependencies-fm"} 2026-08-01T03:54:58Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "ec742bfe-6d10-4ef2-818a-887afa370635", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).ensureNamespace-fm"} 2026-08-01T03:54:58Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "ec742bfe-6d10-4ef2-818a-887afa370635", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).updateReleases-fm"} 2026-08-01T03:54:58Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "ec742bfe-6d10-4ef2-818a-887afa370635", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm"} 2026/08/01 03:54:59 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T03:54:59Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered manifests {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "ec742bfe-6d10-4ef2-818a-887afa370635", "count": 31} 2026/08/01 03:54:59 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T03:54:59Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered runtimes {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "ec742bfe-6d10-4ef2-818a-887afa370635", "count": 15} 2026/08/01 03:54:59 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T03:54:59Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered imagestreams {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "ec742bfe-6d10-4ef2-818a-887afa370635", "count": 3} 2026-08-01T03:54:59Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "ec742bfe-6d10-4ef2-818a-887afa370635", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.NewReconciler.withImmutableFieldRecovery.func9"} 2026-08-01T03:55:00Z ERROR Reconciler error {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "ec742bfe-6d10-4ef2-818a-887afa370635", "error": "provisioning failed: failure deploying resource /torch-distributed: apply failed trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime: unable to patch trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime torch-distributed: Internal error occurred: failed calling webhook \"validator.clustertrainingruntime.trainer.kubeflow.org\": failed to call webhook: Post \"https://kubeflow-trainer-controller-manager.opendatahub.svc:443/validate-trainer-kubeflow-org-v1alpha1-clustertrainingruntime?timeout=10s\": no endpoints available for service \"kubeflow-trainer-controller-manager\""} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313 2026-08-01T03:55:01Z INFO reconcile {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "837b10dc-1434-4f39-a890-eec8f2920b61"} 2026-08-01T03:55:01Z INFO apply {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "837b10dc-1434-4f39-a890-eec8f2920b61"} 2026-08-01T03:55:01Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "837b10dc-1434-4f39-a890-eec8f2920b61", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).checkDependencies-fm"} 2026-08-01T03:55:01Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "837b10dc-1434-4f39-a890-eec8f2920b61", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).ensureNamespace-fm"} 2026-08-01T03:55:01Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "837b10dc-1434-4f39-a890-eec8f2920b61", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).updateReleases-fm"} 2026-08-01T03:55:01Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "837b10dc-1434-4f39-a890-eec8f2920b61", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm"} 2026/08/01 03:55:02 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T03:55:02Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered manifests {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "837b10dc-1434-4f39-a890-eec8f2920b61", "count": 31} 2026/08/01 03:55:02 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T03:55:02Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered runtimes {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "837b10dc-1434-4f39-a890-eec8f2920b61", "count": 15} 2026/08/01 03:55:02 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T03:55:02Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered imagestreams {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "837b10dc-1434-4f39-a890-eec8f2920b61", "count": 3} 2026-08-01T03:55:02Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "837b10dc-1434-4f39-a890-eec8f2920b61", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.NewReconciler.withImmutableFieldRecovery.func9"} 2026-08-01T03:55:02Z ERROR Reconciler error {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "837b10dc-1434-4f39-a890-eec8f2920b61", "error": "provisioning failed: failure deploying resource /torch-distributed: apply failed trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime: unable to patch trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime torch-distributed: Internal error occurred: failed calling webhook \"validator.clustertrainingruntime.trainer.kubeflow.org\": failed to call webhook: Post \"https://kubeflow-trainer-controller-manager.opendatahub.svc:443/validate-trainer-kubeflow-org-v1alpha1-clustertrainingruntime?timeout=10s\": no endpoints available for service \"kubeflow-trainer-controller-manager\""} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313 2026-08-01T03:55:05Z INFO reconcile {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "fcb3ec81-0716-467f-bc2e-5fa4044736ce"} 2026-08-01T03:55:05Z INFO apply {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "fcb3ec81-0716-467f-bc2e-5fa4044736ce"} 2026-08-01T03:55:05Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "fcb3ec81-0716-467f-bc2e-5fa4044736ce", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).checkDependencies-fm"} 2026-08-01T03:55:05Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "fcb3ec81-0716-467f-bc2e-5fa4044736ce", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).ensureNamespace-fm"} 2026-08-01T03:55:05Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "fcb3ec81-0716-467f-bc2e-5fa4044736ce", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).updateReleases-fm"} 2026-08-01T03:55:05Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "fcb3ec81-0716-467f-bc2e-5fa4044736ce", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm"} 2026/08/01 03:55:06 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T03:55:06Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered manifests {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "fcb3ec81-0716-467f-bc2e-5fa4044736ce", "count": 31} 2026/08/01 03:55:06 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T03:55:06Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered runtimes {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "fcb3ec81-0716-467f-bc2e-5fa4044736ce", "count": 15} 2026/08/01 03:55:06 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T03:55:06Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered imagestreams {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "fcb3ec81-0716-467f-bc2e-5fa4044736ce", "count": 3} 2026-08-01T03:55:06Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "fcb3ec81-0716-467f-bc2e-5fa4044736ce", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.NewReconciler.withImmutableFieldRecovery.func9"} 2026-08-01T03:55:06Z ERROR Reconciler error {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "fcb3ec81-0716-467f-bc2e-5fa4044736ce", "error": "provisioning failed: failure deploying resource /torch-distributed: apply failed trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime: unable to patch trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime torch-distributed: Internal error occurred: failed calling webhook \"validator.clustertrainingruntime.trainer.kubeflow.org\": failed to call webhook: Post \"https://kubeflow-trainer-controller-manager.opendatahub.svc:443/validate-trainer-kubeflow-org-v1alpha1-clustertrainingruntime?timeout=10s\": no endpoints available for service \"kubeflow-trainer-controller-manager\""} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313 2026-08-01T03:55:11Z INFO reconcile {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "d8a62dca-70f7-47ea-8e4c-b89017dfd87c"} 2026-08-01T03:55:11Z INFO apply {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "d8a62dca-70f7-47ea-8e4c-b89017dfd87c"} 2026-08-01T03:55:11Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "d8a62dca-70f7-47ea-8e4c-b89017dfd87c", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).checkDependencies-fm"} 2026-08-01T03:55:11Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "d8a62dca-70f7-47ea-8e4c-b89017dfd87c", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).ensureNamespace-fm"} 2026-08-01T03:55:11Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "d8a62dca-70f7-47ea-8e4c-b89017dfd87c", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).updateReleases-fm"} 2026-08-01T03:55:11Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "d8a62dca-70f7-47ea-8e4c-b89017dfd87c", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm"} 2026/08/01 03:55:12 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T03:55:12Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered manifests {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "d8a62dca-70f7-47ea-8e4c-b89017dfd87c", "count": 31} 2026/08/01 03:55:12 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T03:55:12Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered runtimes {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "d8a62dca-70f7-47ea-8e4c-b89017dfd87c", "count": 15} 2026/08/01 03:55:13 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T03:55:13Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered imagestreams {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "d8a62dca-70f7-47ea-8e4c-b89017dfd87c", "count": 3} 2026-08-01T03:55:13Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "d8a62dca-70f7-47ea-8e4c-b89017dfd87c", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.NewReconciler.withImmutableFieldRecovery.func9"} 2026-08-01T03:55:13Z ERROR Reconciler error {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "d8a62dca-70f7-47ea-8e4c-b89017dfd87c", "error": "provisioning failed: failure deploying resource /torch-distributed: apply failed trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime: unable to patch trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime torch-distributed: Internal error occurred: failed calling webhook \"validator.clustertrainingruntime.trainer.kubeflow.org\": failed to call webhook: Post \"https://kubeflow-trainer-controller-manager.opendatahub.svc:443/validate-trainer-kubeflow-org-v1alpha1-clustertrainingruntime?timeout=10s\": no endpoints available for service \"kubeflow-trainer-controller-manager\""} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313 2026-08-01T03:55:23Z INFO reconcile {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "b85608ec-83fc-4618-9e66-12c859ca5397"} 2026-08-01T03:55:23Z INFO apply {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "b85608ec-83fc-4618-9e66-12c859ca5397"} 2026-08-01T03:55:23Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "b85608ec-83fc-4618-9e66-12c859ca5397", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).checkDependencies-fm"} 2026-08-01T03:55:23Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "b85608ec-83fc-4618-9e66-12c859ca5397", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).ensureNamespace-fm"} 2026-08-01T03:55:23Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "b85608ec-83fc-4618-9e66-12c859ca5397", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).updateReleases-fm"} 2026-08-01T03:55:23Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "b85608ec-83fc-4618-9e66-12c859ca5397", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm"} 2026/08/01 03:55:24 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T03:55:24Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered manifests {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "b85608ec-83fc-4618-9e66-12c859ca5397", "count": 31} 2026/08/01 03:55:24 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T03:55:24Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered runtimes {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "b85608ec-83fc-4618-9e66-12c859ca5397", "count": 15} 2026/08/01 03:55:24 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T03:55:24Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered imagestreams {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "b85608ec-83fc-4618-9e66-12c859ca5397", "count": 3} 2026-08-01T03:55:24Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "b85608ec-83fc-4618-9e66-12c859ca5397", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.NewReconciler.withImmutableFieldRecovery.func9"} 2026-08-01T03:55:25Z ERROR Reconciler error {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "b85608ec-83fc-4618-9e66-12c859ca5397", "error": "provisioning failed: failure deploying resource /torch-distributed: apply failed trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime: unable to patch trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime torch-distributed: Internal error occurred: failed calling webhook \"validator.clustertrainingruntime.trainer.kubeflow.org\": failed to call webhook: Post \"https://kubeflow-trainer-controller-manager.opendatahub.svc:443/validate-trainer-kubeflow-org-v1alpha1-clustertrainingruntime?timeout=10s\": no endpoints available for service \"kubeflow-trainer-controller-manager\""} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313 2026-08-01T03:55:45Z INFO reconcile {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "07c2106b-6caf-411c-a281-5e5de80c9b2d"} 2026-08-01T03:55:45Z INFO apply {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "07c2106b-6caf-411c-a281-5e5de80c9b2d"} 2026-08-01T03:55:45Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "07c2106b-6caf-411c-a281-5e5de80c9b2d", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).checkDependencies-fm"} 2026-08-01T03:55:45Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "07c2106b-6caf-411c-a281-5e5de80c9b2d", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).ensureNamespace-fm"} 2026-08-01T03:55:45Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "07c2106b-6caf-411c-a281-5e5de80c9b2d", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).updateReleases-fm"} 2026-08-01T03:55:45Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "07c2106b-6caf-411c-a281-5e5de80c9b2d", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm"} 2026/08/01 03:55:46 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T03:55:46Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered manifests {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "07c2106b-6caf-411c-a281-5e5de80c9b2d", "count": 31} 2026/08/01 03:55:46 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T03:55:46Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered runtimes {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "07c2106b-6caf-411c-a281-5e5de80c9b2d", "count": 15} 2026/08/01 03:55:46 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T03:55:46Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered imagestreams {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "07c2106b-6caf-411c-a281-5e5de80c9b2d", "count": 3} 2026-08-01T03:55:46Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "07c2106b-6caf-411c-a281-5e5de80c9b2d", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.NewReconciler.withImmutableFieldRecovery.func9"} 2026-08-01T03:55:47Z ERROR Reconciler error {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "07c2106b-6caf-411c-a281-5e5de80c9b2d", "error": "provisioning failed: failure deploying resource /torch-distributed: apply failed trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime: unable to patch trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime torch-distributed: Internal error occurred: failed calling webhook \"validator.clustertrainingruntime.trainer.kubeflow.org\": failed to call webhook: Post \"https://kubeflow-trainer-controller-manager.opendatahub.svc:443/validate-trainer-kubeflow-org-v1alpha1-clustertrainingruntime?timeout=10s\": no endpoints available for service \"kubeflow-trainer-controller-manager\""} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313 2026-08-01T03:56:27Z INFO reconcile {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "ec961bda-3cdc-4a39-b090-1c84cf5fe470"} 2026-08-01T03:56:27Z INFO apply {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "ec961bda-3cdc-4a39-b090-1c84cf5fe470"} 2026-08-01T03:56:27Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "ec961bda-3cdc-4a39-b090-1c84cf5fe470", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).checkDependencies-fm"} 2026-08-01T03:56:28Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "ec961bda-3cdc-4a39-b090-1c84cf5fe470", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).ensureNamespace-fm"} 2026-08-01T03:56:28Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "ec961bda-3cdc-4a39-b090-1c84cf5fe470", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).updateReleases-fm"} 2026-08-01T03:56:28Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "ec961bda-3cdc-4a39-b090-1c84cf5fe470", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm"} 2026/08/01 03:56:29 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T03:56:29Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered manifests {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "ec961bda-3cdc-4a39-b090-1c84cf5fe470", "count": 31} 2026/08/01 03:56:29 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T03:56:29Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered runtimes {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "ec961bda-3cdc-4a39-b090-1c84cf5fe470", "count": 15} 2026/08/01 03:56:29 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T03:56:29Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered imagestreams {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "ec961bda-3cdc-4a39-b090-1c84cf5fe470", "count": 3} 2026-08-01T03:56:29Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "ec961bda-3cdc-4a39-b090-1c84cf5fe470", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.NewReconciler.withImmutableFieldRecovery.func9"} 2026-08-01T03:56:29Z ERROR Reconciler error {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "ec961bda-3cdc-4a39-b090-1c84cf5fe470", "error": "provisioning failed: failure deploying resource /torch-distributed: apply failed trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime: unable to patch trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime torch-distributed: Internal error occurred: failed calling webhook \"validator.clustertrainingruntime.trainer.kubeflow.org\": failed to call webhook: Post \"https://kubeflow-trainer-controller-manager.opendatahub.svc:443/validate-trainer-kubeflow-org-v1alpha1-clustertrainingruntime?timeout=10s\": no endpoints available for service \"kubeflow-trainer-controller-manager\""} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313 client.go:143: Kubernetes events: 0001-01-01T00:00:00Z Normal Scheduled trainer-operator-controller-manager-69855b4775-82jzv Successfully assigned trainer-operator-system/trainer-operator-controller-manager-69855b4775-82jzv to ip-10-0-128-255.ec2.internal 2026-08-01T03:53:52Z Normal AddedInterface trainer-operator-controller-manager-69855b4775-82jzv Add eth0 [10.134.0.26/23] from ovn-kubernetes 2026-08-01T03:53:52Z Normal Pulling trainer-operator-controller-manager-69855b4775-82jzv Pulling image "quay.io/opendatahub/odh-trainer-operator@sha256:33a5eca3769674e41aa4fa0bb36599289f059c0640da5906b0aa990a0e93e627" 2026-08-01T03:53:52Z Normal SuccessfulCreate trainer-operator-controller-manager-69855b4775 Created pod: trainer-operator-controller-manager-69855b4775-82jzv 2026-08-01T03:53:52Z Normal ScalingReplicaSet trainer-operator-controller-manager Scaled up replica set trainer-operator-controller-manager-69855b4775 from 0 to 1 2026-08-01T03:53:54Z Normal Pulled trainer-operator-controller-manager-69855b4775-82jzv Successfully pulled image "quay.io/opendatahub/odh-trainer-operator@sha256:33a5eca3769674e41aa4fa0bb36599289f059c0640da5906b0aa990a0e93e627" in 2.205s (2.205s including waiting). Image size: 197968060 bytes. 2026-08-01T03:53:54Z Normal Created trainer-operator-controller-manager-69855b4775-82jzv Created container: manager 2026-08-01T03:53:54Z Normal Started trainer-operator-controller-manager-69855b4775-82jzv Started container manager --- FAIL: TestTrainJobWithClusterTrainingRuntime (121.08s) FAIL FAIL github.com/opendatahub-io/trainer-operator/test/e2e/ocp 121.131s FAIL make: *** [Makefile:151: test-e2e-ocp] Error 1