go test ./test/e2e/ocp/ -v -timeout 30m Starting OCP e2e test suite === RUN TestTrainJobWithClusterTrainingRuntime trainjob_test.go:53: Timed out after 120.000s. The function passed to Eventually failed at /workspace/source/src_code/test/e2e/ocp/trainjob_test.go:52 with: Expected : Not Ready to equal : Ready client.go:143: Controller logs: 2026-08-01T05:49:09Z INFO setup starting manager 2026-08-01T05:49:09Z INFO controller-runtime.metrics Starting metrics server 2026-08-01T05:49:09Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": ":8080", "secure": false} 2026-08-01T05:49:09Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2026-08-01T05:49:09Z INFO Starting EventSource {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "source": "kind source: *unstructured.Unstructured"} 2026-08-01T05:49:09Z INFO Starting EventSource {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "source": "kind source: *unstructured.Unstructured"} 2026-08-01T05:49:09Z INFO Starting EventSource {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "source": "kind source: *unstructured.Unstructured"} 2026-08-01T05:49:09Z INFO Starting EventSource {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "source": "kind source: *unstructured.Unstructured"} 2026-08-01T05:49:09Z INFO Starting EventSource {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "source": "kind source: *unstructured.Unstructured"} 2026-08-01T05:49:10Z INFO Starting Controller {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer"} 2026-08-01T05:49:10Z INFO Starting workers {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "worker count": 1} 2026-08-01T05:49:57Z INFO reconcile {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "00ed58c7-2519-4a1a-87a1-2e834dae0e64"} 2026-08-01T05:49:57Z INFO adding finalizer {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "00ed58c7-2519-4a1a-87a1-2e834dae0e64"} 2026-08-01T05:49:57Z INFO apply {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "00ed58c7-2519-4a1a-87a1-2e834dae0e64"} 2026-08-01T05:49:57Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "00ed58c7-2519-4a1a-87a1-2e834dae0e64", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).checkDependencies-fm"} 2026-08-01T05:49:57Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "00ed58c7-2519-4a1a-87a1-2e834dae0e64", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).ensureNamespace-fm"} 2026-08-01T05:49:57Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).ensureNamespace-fm Created namespace {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "00ed58c7-2519-4a1a-87a1-2e834dae0e64", "namespace": "opendatahub"} 2026-08-01T05:49:57Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "00ed58c7-2519-4a1a-87a1-2e834dae0e64", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).updateReleases-fm"} 2026-08-01T05:49:57Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "00ed58c7-2519-4a1a-87a1-2e834dae0e64", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm"} 2026/08/01 05:49:58 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T05:49:58Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered manifests {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "00ed58c7-2519-4a1a-87a1-2e834dae0e64", "count": 31} 2026/08/01 05:49:58 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T05:49:58Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered runtimes {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "00ed58c7-2519-4a1a-87a1-2e834dae0e64", "count": 15} 2026/08/01 05:49:58 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T05:49:58Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered imagestreams {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "00ed58c7-2519-4a1a-87a1-2e834dae0e64", "count": 3} 2026-08-01T05:49:58Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "00ed58c7-2519-4a1a-87a1-2e834dae0e64", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.NewReconciler.withImmutableFieldRecovery.func9"} 2026-08-01T05:49:59Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "00ed58c7-2519-4a1a-87a1-2e834dae0e64", "action": "github.com/opendatahub-io/odh-platform-utilities/framework/controller/actions/status/deployments.(*Action).run-fm"} 2026-08-01T05:50:00Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "00ed58c7-2519-4a1a-87a1-2e834dae0e64", "action": "github.com/opendatahub-io/odh-platform-utilities/framework/controller/actions/gc.(*Action).run-fm"} 2026-08-01T05:50:01Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "00ed58c7-2519-4a1a-87a1-2e834dae0e64", "action": "github.com/opendatahub-io/odh-platform-utilities/framework/controller/reconciler.(*dynamicWatchAction).run-fm"} 2026-08-01T05:50:02Z INFO Starting EventSource {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "source": "kind source: *unstructured.Unstructured"} 2026-08-01T05:50:02Z INFO reconcile {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "eccdf62a-0a1d-4161-9696-54bf2a118773"} 2026-08-01T05:50:02Z INFO apply {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "eccdf62a-0a1d-4161-9696-54bf2a118773"} 2026-08-01T05:50:02Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "eccdf62a-0a1d-4161-9696-54bf2a118773", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).checkDependencies-fm"} 2026-08-01T05:50:02Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "eccdf62a-0a1d-4161-9696-54bf2a118773", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).ensureNamespace-fm"} 2026-08-01T05:50:02Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "eccdf62a-0a1d-4161-9696-54bf2a118773", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).updateReleases-fm"} 2026-08-01T05:50:02Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "eccdf62a-0a1d-4161-9696-54bf2a118773", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm"} 2026/08/01 05:50:03 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T05:50:03Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered manifests {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "eccdf62a-0a1d-4161-9696-54bf2a118773", "count": 31} 2026/08/01 05:50:03 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T05:50:03Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered runtimes {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "eccdf62a-0a1d-4161-9696-54bf2a118773", "count": 15} 2026/08/01 05:50:03 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T05:50:03Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered imagestreams {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "eccdf62a-0a1d-4161-9696-54bf2a118773", "count": 3} 2026-08-01T05:50:03Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "eccdf62a-0a1d-4161-9696-54bf2a118773", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.NewReconciler.withImmutableFieldRecovery.func9"} 2026-08-01T05:50:04Z ERROR Reconciler error {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "eccdf62a-0a1d-4161-9696-54bf2a118773", "error": "provisioning failed: failure deploying resource /torch-distributed: apply failed trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime: unable to patch trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime torch-distributed: Internal error occurred: failed calling webhook \"validator.clustertrainingruntime.trainer.kubeflow.org\": failed to call webhook: Post \"https://kubeflow-trainer-controller-manager.opendatahub.svc:443/validate-trainer-kubeflow-org-v1alpha1-clustertrainingruntime?timeout=10s\": no endpoints available for service \"kubeflow-trainer-controller-manager\""} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313 2026-08-01T05:50:04Z INFO reconcile {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "b679dd7e-86aa-4f98-9fad-9da57c465d04"} 2026-08-01T05:50:04Z INFO apply {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "b679dd7e-86aa-4f98-9fad-9da57c465d04"} 2026-08-01T05:50:04Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "b679dd7e-86aa-4f98-9fad-9da57c465d04", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).checkDependencies-fm"} 2026-08-01T05:50:04Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "b679dd7e-86aa-4f98-9fad-9da57c465d04", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).ensureNamespace-fm"} 2026-08-01T05:50:04Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "b679dd7e-86aa-4f98-9fad-9da57c465d04", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).updateReleases-fm"} 2026-08-01T05:50:04Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "b679dd7e-86aa-4f98-9fad-9da57c465d04", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm"} 2026/08/01 05:50:05 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T05:50:05Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered manifests {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "b679dd7e-86aa-4f98-9fad-9da57c465d04", "count": 31} 2026/08/01 05:50:05 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T05:50:05Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered runtimes {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "b679dd7e-86aa-4f98-9fad-9da57c465d04", "count": 15} 2026/08/01 05:50:05 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T05:50:05Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered imagestreams {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "b679dd7e-86aa-4f98-9fad-9da57c465d04", "count": 3} 2026-08-01T05:50:05Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "b679dd7e-86aa-4f98-9fad-9da57c465d04", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.NewReconciler.withImmutableFieldRecovery.func9"} 2026-08-01T05:50:05Z ERROR Reconciler error {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "b679dd7e-86aa-4f98-9fad-9da57c465d04", "error": "provisioning failed: failure deploying resource /torch-distributed: apply failed trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime: unable to patch trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime torch-distributed: Internal error occurred: failed calling webhook \"validator.clustertrainingruntime.trainer.kubeflow.org\": failed to call webhook: Post \"https://kubeflow-trainer-controller-manager.opendatahub.svc:443/validate-trainer-kubeflow-org-v1alpha1-clustertrainingruntime?timeout=10s\": no endpoints available for service \"kubeflow-trainer-controller-manager\""} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313 2026-08-01T05:50:05Z INFO reconcile {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "dd2e3cdf-cf6e-4ac0-af4b-75c590254ac1"} 2026-08-01T05:50:05Z INFO apply {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "dd2e3cdf-cf6e-4ac0-af4b-75c590254ac1"} 2026-08-01T05:50:05Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "dd2e3cdf-cf6e-4ac0-af4b-75c590254ac1", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).checkDependencies-fm"} 2026-08-01T05:50:05Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "dd2e3cdf-cf6e-4ac0-af4b-75c590254ac1", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).ensureNamespace-fm"} 2026-08-01T05:50:05Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "dd2e3cdf-cf6e-4ac0-af4b-75c590254ac1", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).updateReleases-fm"} 2026-08-01T05:50:05Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "dd2e3cdf-cf6e-4ac0-af4b-75c590254ac1", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm"} 2026/08/01 05:50:06 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T05:50:06Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered manifests {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "dd2e3cdf-cf6e-4ac0-af4b-75c590254ac1", "count": 31} 2026/08/01 05:50:06 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T05:50:06Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered runtimes {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "dd2e3cdf-cf6e-4ac0-af4b-75c590254ac1", "count": 15} 2026/08/01 05:50:06 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T05:50:06Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered imagestreams {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "dd2e3cdf-cf6e-4ac0-af4b-75c590254ac1", "count": 3} 2026-08-01T05:50:06Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "dd2e3cdf-cf6e-4ac0-af4b-75c590254ac1", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.NewReconciler.withImmutableFieldRecovery.func9"} 2026-08-01T05:50:06Z ERROR Reconciler error {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "dd2e3cdf-cf6e-4ac0-af4b-75c590254ac1", "error": "provisioning failed: failure deploying resource /torch-distributed: apply failed trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime: unable to patch trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime torch-distributed: Internal error occurred: failed calling webhook \"validator.clustertrainingruntime.trainer.kubeflow.org\": failed to call webhook: Post \"https://kubeflow-trainer-controller-manager.opendatahub.svc:443/validate-trainer-kubeflow-org-v1alpha1-clustertrainingruntime?timeout=10s\": no endpoints available for service \"kubeflow-trainer-controller-manager\""} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313 2026-08-01T05:50:06Z INFO reconcile {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "5f055160-11bc-482b-befe-bee2bd03c264"} 2026-08-01T05:50:06Z INFO apply {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "5f055160-11bc-482b-befe-bee2bd03c264"} 2026-08-01T05:50:06Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "5f055160-11bc-482b-befe-bee2bd03c264", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).checkDependencies-fm"} 2026-08-01T05:50:06Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "5f055160-11bc-482b-befe-bee2bd03c264", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).ensureNamespace-fm"} 2026-08-01T05:50:06Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "5f055160-11bc-482b-befe-bee2bd03c264", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).updateReleases-fm"} 2026-08-01T05:50:06Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "5f055160-11bc-482b-befe-bee2bd03c264", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm"} 2026/08/01 05:50:07 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T05:50:07Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered manifests {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "5f055160-11bc-482b-befe-bee2bd03c264", "count": 31} 2026/08/01 05:50:07 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T05:50:07Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered runtimes {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "5f055160-11bc-482b-befe-bee2bd03c264", "count": 15} 2026/08/01 05:50:07 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T05:50:07Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered imagestreams {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "5f055160-11bc-482b-befe-bee2bd03c264", "count": 3} 2026-08-01T05:50:07Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "5f055160-11bc-482b-befe-bee2bd03c264", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.NewReconciler.withImmutableFieldRecovery.func9"} 2026-08-01T05:50:08Z ERROR Reconciler error {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "5f055160-11bc-482b-befe-bee2bd03c264", "error": "provisioning failed: failure deploying resource /torch-distributed: apply failed trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime: unable to patch trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime torch-distributed: Internal error occurred: failed calling webhook \"validator.clustertrainingruntime.trainer.kubeflow.org\": failed to call webhook: Post \"https://kubeflow-trainer-controller-manager.opendatahub.svc:443/validate-trainer-kubeflow-org-v1alpha1-clustertrainingruntime?timeout=10s\": no endpoints available for service \"kubeflow-trainer-controller-manager\""} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313 2026-08-01T05:50:08Z INFO reconcile {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "0fdea856-34e2-412c-adc7-36c7189b6835"} 2026-08-01T05:50:08Z INFO apply {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "0fdea856-34e2-412c-adc7-36c7189b6835"} 2026-08-01T05:50:08Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "0fdea856-34e2-412c-adc7-36c7189b6835", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).checkDependencies-fm"} 2026-08-01T05:50:08Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "0fdea856-34e2-412c-adc7-36c7189b6835", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).ensureNamespace-fm"} 2026-08-01T05:50:08Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "0fdea856-34e2-412c-adc7-36c7189b6835", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).updateReleases-fm"} 2026-08-01T05:50:08Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "0fdea856-34e2-412c-adc7-36c7189b6835", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm"} 2026/08/01 05:50:09 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T05:50:09Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered manifests {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "0fdea856-34e2-412c-adc7-36c7189b6835", "count": 31} 2026/08/01 05:50:09 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T05:50:09Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered runtimes {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "0fdea856-34e2-412c-adc7-36c7189b6835", "count": 15} 2026/08/01 05:50:09 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T05:50:09Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered imagestreams {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "0fdea856-34e2-412c-adc7-36c7189b6835", "count": 3} 2026-08-01T05:50:09Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "0fdea856-34e2-412c-adc7-36c7189b6835", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.NewReconciler.withImmutableFieldRecovery.func9"} 2026-08-01T05:50:09Z ERROR Reconciler error {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "0fdea856-34e2-412c-adc7-36c7189b6835", "error": "provisioning failed: failure deploying resource /torch-distributed: apply failed trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime: unable to patch trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime torch-distributed: Internal error occurred: failed calling webhook \"validator.clustertrainingruntime.trainer.kubeflow.org\": failed to call webhook: Post \"https://kubeflow-trainer-controller-manager.opendatahub.svc:443/validate-trainer-kubeflow-org-v1alpha1-clustertrainingruntime?timeout=10s\": no endpoints available for service \"kubeflow-trainer-controller-manager\""} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313 2026-08-01T05:50:09Z INFO reconcile {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "fc0f0512-20c5-48bd-97ac-8e445f8ae5dc"} 2026-08-01T05:50:09Z INFO apply {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "fc0f0512-20c5-48bd-97ac-8e445f8ae5dc"} 2026-08-01T05:50:09Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "fc0f0512-20c5-48bd-97ac-8e445f8ae5dc", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).checkDependencies-fm"} 2026-08-01T05:50:09Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "fc0f0512-20c5-48bd-97ac-8e445f8ae5dc", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).ensureNamespace-fm"} 2026-08-01T05:50:09Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "fc0f0512-20c5-48bd-97ac-8e445f8ae5dc", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).updateReleases-fm"} 2026-08-01T05:50:09Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "fc0f0512-20c5-48bd-97ac-8e445f8ae5dc", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm"} 2026/08/01 05:50:10 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T05:50:10Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered manifests {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "fc0f0512-20c5-48bd-97ac-8e445f8ae5dc", "count": 31} 2026/08/01 05:50:10 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T05:50:10Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered runtimes {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "fc0f0512-20c5-48bd-97ac-8e445f8ae5dc", "count": 15} 2026/08/01 05:50:10 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T05:50:10Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered imagestreams {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "fc0f0512-20c5-48bd-97ac-8e445f8ae5dc", "count": 3} 2026-08-01T05:50:10Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "fc0f0512-20c5-48bd-97ac-8e445f8ae5dc", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.NewReconciler.withImmutableFieldRecovery.func9"} 2026-08-01T05:50:11Z ERROR Reconciler error {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "fc0f0512-20c5-48bd-97ac-8e445f8ae5dc", "error": "provisioning failed: failure deploying resource /torch-distributed: apply failed trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime: unable to patch trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime torch-distributed: Internal error occurred: failed calling webhook \"validator.clustertrainingruntime.trainer.kubeflow.org\": failed to call webhook: Post \"https://kubeflow-trainer-controller-manager.opendatahub.svc:443/validate-trainer-kubeflow-org-v1alpha1-clustertrainingruntime?timeout=10s\": no endpoints available for service \"kubeflow-trainer-controller-manager\""} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313 2026-08-01T05:50:11Z INFO reconcile {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "b39eb43d-1dfe-4fb1-9830-8628984496a2"} 2026-08-01T05:50:11Z INFO apply {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "b39eb43d-1dfe-4fb1-9830-8628984496a2"} 2026-08-01T05:50:11Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "b39eb43d-1dfe-4fb1-9830-8628984496a2", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).checkDependencies-fm"} 2026-08-01T05:50:11Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "b39eb43d-1dfe-4fb1-9830-8628984496a2", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).ensureNamespace-fm"} 2026-08-01T05:50:11Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "b39eb43d-1dfe-4fb1-9830-8628984496a2", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).updateReleases-fm"} 2026-08-01T05:50:11Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "b39eb43d-1dfe-4fb1-9830-8628984496a2", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm"} 2026/08/01 05:50:12 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T05:50:12Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered manifests {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "b39eb43d-1dfe-4fb1-9830-8628984496a2", "count": 31} 2026/08/01 05:50:12 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T05:50:12Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered runtimes {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "b39eb43d-1dfe-4fb1-9830-8628984496a2", "count": 15} 2026/08/01 05:50:12 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T05:50:12Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered imagestreams {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "b39eb43d-1dfe-4fb1-9830-8628984496a2", "count": 3} 2026-08-01T05:50:12Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "b39eb43d-1dfe-4fb1-9830-8628984496a2", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.NewReconciler.withImmutableFieldRecovery.func9"} 2026-08-01T05:50:12Z ERROR Reconciler error {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "b39eb43d-1dfe-4fb1-9830-8628984496a2", "error": "provisioning failed: failure deploying resource /torch-distributed: apply failed trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime: unable to patch trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime torch-distributed: Internal error occurred: failed calling webhook \"validator.clustertrainingruntime.trainer.kubeflow.org\": failed to call webhook: Post \"https://kubeflow-trainer-controller-manager.opendatahub.svc:443/validate-trainer-kubeflow-org-v1alpha1-clustertrainingruntime?timeout=10s\": no endpoints available for service \"kubeflow-trainer-controller-manager\""} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313 2026-08-01T05:50:13Z INFO reconcile {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "408e7391-74d9-4c2e-9d33-3c779e71fc05"} 2026-08-01T05:50:13Z INFO apply {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "408e7391-74d9-4c2e-9d33-3c779e71fc05"} 2026-08-01T05:50:13Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "408e7391-74d9-4c2e-9d33-3c779e71fc05", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).checkDependencies-fm"} 2026-08-01T05:50:13Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "408e7391-74d9-4c2e-9d33-3c779e71fc05", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).ensureNamespace-fm"} 2026-08-01T05:50:13Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "408e7391-74d9-4c2e-9d33-3c779e71fc05", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).updateReleases-fm"} 2026-08-01T05:50:13Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "408e7391-74d9-4c2e-9d33-3c779e71fc05", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm"} 2026/08/01 05:50:14 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T05:50:14Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered manifests {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "408e7391-74d9-4c2e-9d33-3c779e71fc05", "count": 31} 2026/08/01 05:50:14 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T05:50:14Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered runtimes {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "408e7391-74d9-4c2e-9d33-3c779e71fc05", "count": 15} 2026/08/01 05:50:14 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T05:50:14Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered imagestreams {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "408e7391-74d9-4c2e-9d33-3c779e71fc05", "count": 3} 2026-08-01T05:50:14Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "408e7391-74d9-4c2e-9d33-3c779e71fc05", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.NewReconciler.withImmutableFieldRecovery.func9"} 2026-08-01T05:50:14Z ERROR Reconciler error {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "408e7391-74d9-4c2e-9d33-3c779e71fc05", "error": "provisioning failed: failure deploying resource /torch-distributed: apply failed trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime: unable to patch trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime torch-distributed: Internal error occurred: failed calling webhook \"validator.clustertrainingruntime.trainer.kubeflow.org\": failed to call webhook: Post \"https://kubeflow-trainer-controller-manager.opendatahub.svc:443/validate-trainer-kubeflow-org-v1alpha1-clustertrainingruntime?timeout=10s\": no endpoints available for service \"kubeflow-trainer-controller-manager\""} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313 2026-08-01T05:50:15Z INFO reconcile {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "1c357d15-940e-487f-a04d-c2ac83eee82b"} 2026-08-01T05:50:15Z INFO apply {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "1c357d15-940e-487f-a04d-c2ac83eee82b"} 2026-08-01T05:50:15Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "1c357d15-940e-487f-a04d-c2ac83eee82b", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).checkDependencies-fm"} 2026-08-01T05:50:15Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "1c357d15-940e-487f-a04d-c2ac83eee82b", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).ensureNamespace-fm"} 2026-08-01T05:50:15Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "1c357d15-940e-487f-a04d-c2ac83eee82b", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).updateReleases-fm"} 2026-08-01T05:50:15Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "1c357d15-940e-487f-a04d-c2ac83eee82b", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm"} 2026/08/01 05:50:16 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T05:50:16Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered manifests {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "1c357d15-940e-487f-a04d-c2ac83eee82b", "count": 31} 2026/08/01 05:50:16 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T05:50:16Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered runtimes {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "1c357d15-940e-487f-a04d-c2ac83eee82b", "count": 15} 2026/08/01 05:50:16 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T05:50:16Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered imagestreams {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "1c357d15-940e-487f-a04d-c2ac83eee82b", "count": 3} 2026-08-01T05:50:16Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "1c357d15-940e-487f-a04d-c2ac83eee82b", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.NewReconciler.withImmutableFieldRecovery.func9"} 2026-08-01T05:50:16Z ERROR Reconciler error {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "1c357d15-940e-487f-a04d-c2ac83eee82b", "error": "provisioning failed: failure deploying resource /torch-distributed: apply failed trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime: unable to patch trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime torch-distributed: Internal error occurred: failed calling webhook \"validator.clustertrainingruntime.trainer.kubeflow.org\": failed to call webhook: Post \"https://kubeflow-trainer-controller-manager.opendatahub.svc:443/validate-trainer-kubeflow-org-v1alpha1-clustertrainingruntime?timeout=10s\": no endpoints available for service \"kubeflow-trainer-controller-manager\""} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313 2026-08-01T05:50:17Z INFO reconcile {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "0aff0ffe-6156-4213-9399-5b4560479380"} 2026-08-01T05:50:17Z INFO apply {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "0aff0ffe-6156-4213-9399-5b4560479380"} 2026-08-01T05:50:17Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "0aff0ffe-6156-4213-9399-5b4560479380", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).checkDependencies-fm"} 2026-08-01T05:50:17Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "0aff0ffe-6156-4213-9399-5b4560479380", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).ensureNamespace-fm"} 2026-08-01T05:50:17Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "0aff0ffe-6156-4213-9399-5b4560479380", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).updateReleases-fm"} 2026-08-01T05:50:17Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "0aff0ffe-6156-4213-9399-5b4560479380", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm"} 2026/08/01 05:50:18 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T05:50:18Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered manifests {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "0aff0ffe-6156-4213-9399-5b4560479380", "count": 31} 2026/08/01 05:50:18 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T05:50:18Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered runtimes {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "0aff0ffe-6156-4213-9399-5b4560479380", "count": 15} 2026/08/01 05:50:18 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T05:50:18Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered imagestreams {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "0aff0ffe-6156-4213-9399-5b4560479380", "count": 3} 2026-08-01T05:50:18Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "0aff0ffe-6156-4213-9399-5b4560479380", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.NewReconciler.withImmutableFieldRecovery.func9"} 2026-08-01T05:50:19Z ERROR Reconciler error {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "0aff0ffe-6156-4213-9399-5b4560479380", "error": "provisioning failed: failure deploying resource /torch-distributed: apply failed trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime: unable to patch trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime torch-distributed: Internal error occurred: failed calling webhook \"validator.clustertrainingruntime.trainer.kubeflow.org\": failed to call webhook: Post \"https://kubeflow-trainer-controller-manager.opendatahub.svc:443/validate-trainer-kubeflow-org-v1alpha1-clustertrainingruntime?timeout=10s\": no endpoints available for service \"kubeflow-trainer-controller-manager\""} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313 2026-08-01T05:50:21Z INFO reconcile {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "2c18e333-f76d-499b-a21a-49a97b5a2b16"} 2026-08-01T05:50:21Z INFO apply {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "2c18e333-f76d-499b-a21a-49a97b5a2b16"} 2026-08-01T05:50:21Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "2c18e333-f76d-499b-a21a-49a97b5a2b16", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).checkDependencies-fm"} 2026-08-01T05:50:21Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "2c18e333-f76d-499b-a21a-49a97b5a2b16", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).ensureNamespace-fm"} 2026-08-01T05:50:21Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "2c18e333-f76d-499b-a21a-49a97b5a2b16", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).updateReleases-fm"} 2026-08-01T05:50:21Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "2c18e333-f76d-499b-a21a-49a97b5a2b16", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm"} 2026/08/01 05:50:22 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T05:50:22Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered manifests {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "2c18e333-f76d-499b-a21a-49a97b5a2b16", "count": 31} 2026/08/01 05:50:22 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T05:50:22Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered runtimes {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "2c18e333-f76d-499b-a21a-49a97b5a2b16", "count": 15} 2026/08/01 05:50:22 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T05:50:22Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered imagestreams {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "2c18e333-f76d-499b-a21a-49a97b5a2b16", "count": 3} 2026-08-01T05:50:22Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "2c18e333-f76d-499b-a21a-49a97b5a2b16", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.NewReconciler.withImmutableFieldRecovery.func9"} 2026-08-01T05:50:22Z ERROR Reconciler error {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "2c18e333-f76d-499b-a21a-49a97b5a2b16", "error": "provisioning failed: failure deploying resource /torch-distributed: apply failed trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime: unable to patch trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime torch-distributed: Internal error occurred: failed calling webhook \"validator.clustertrainingruntime.trainer.kubeflow.org\": failed to call webhook: Post \"https://kubeflow-trainer-controller-manager.opendatahub.svc:443/validate-trainer-kubeflow-org-v1alpha1-clustertrainingruntime?timeout=10s\": no endpoints available for service \"kubeflow-trainer-controller-manager\""} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313 2026-08-01T05:50:28Z INFO reconcile {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "bc5e8069-e24e-4a72-aebc-a4a8c2b09843"} 2026-08-01T05:50:28Z INFO apply {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "bc5e8069-e24e-4a72-aebc-a4a8c2b09843"} 2026-08-01T05:50:28Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "bc5e8069-e24e-4a72-aebc-a4a8c2b09843", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).checkDependencies-fm"} 2026-08-01T05:50:28Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "bc5e8069-e24e-4a72-aebc-a4a8c2b09843", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).ensureNamespace-fm"} 2026-08-01T05:50:28Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "bc5e8069-e24e-4a72-aebc-a4a8c2b09843", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).updateReleases-fm"} 2026-08-01T05:50:28Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "bc5e8069-e24e-4a72-aebc-a4a8c2b09843", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm"} 2026/08/01 05:50:29 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T05:50:29Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered manifests {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "bc5e8069-e24e-4a72-aebc-a4a8c2b09843", "count": 31} 2026/08/01 05:50:29 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T05:50:29Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered runtimes {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "bc5e8069-e24e-4a72-aebc-a4a8c2b09843", "count": 15} 2026/08/01 05:50:29 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T05:50:29Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered imagestreams {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "bc5e8069-e24e-4a72-aebc-a4a8c2b09843", "count": 3} 2026-08-01T05:50:29Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "bc5e8069-e24e-4a72-aebc-a4a8c2b09843", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.NewReconciler.withImmutableFieldRecovery.func9"} 2026-08-01T05:50:29Z ERROR Reconciler error {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "bc5e8069-e24e-4a72-aebc-a4a8c2b09843", "error": "provisioning failed: failure deploying resource /torch-distributed: apply failed trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime: unable to patch trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime torch-distributed: Internal error occurred: failed calling webhook \"validator.clustertrainingruntime.trainer.kubeflow.org\": failed to call webhook: Post \"https://kubeflow-trainer-controller-manager.opendatahub.svc:443/validate-trainer-kubeflow-org-v1alpha1-clustertrainingruntime?timeout=10s\": no endpoints available for service \"kubeflow-trainer-controller-manager\""} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313 2026-08-01T05:50:39Z INFO reconcile {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "1d8b8b5a-8fbe-435f-b1f9-acf43017c204"} 2026-08-01T05:50:39Z INFO apply {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "1d8b8b5a-8fbe-435f-b1f9-acf43017c204"} 2026-08-01T05:50:39Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "1d8b8b5a-8fbe-435f-b1f9-acf43017c204", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).checkDependencies-fm"} 2026-08-01T05:50:39Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "1d8b8b5a-8fbe-435f-b1f9-acf43017c204", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).ensureNamespace-fm"} 2026-08-01T05:50:39Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "1d8b8b5a-8fbe-435f-b1f9-acf43017c204", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).updateReleases-fm"} 2026-08-01T05:50:39Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "1d8b8b5a-8fbe-435f-b1f9-acf43017c204", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm"} 2026/08/01 05:50:40 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T05:50:40Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered manifests {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "1d8b8b5a-8fbe-435f-b1f9-acf43017c204", "count": 31} 2026/08/01 05:50:40 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T05:50:40Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered runtimes {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "1d8b8b5a-8fbe-435f-b1f9-acf43017c204", "count": 15} 2026/08/01 05:50:40 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T05:50:40Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered imagestreams {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "1d8b8b5a-8fbe-435f-b1f9-acf43017c204", "count": 3} 2026-08-01T05:50:40Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "1d8b8b5a-8fbe-435f-b1f9-acf43017c204", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.NewReconciler.withImmutableFieldRecovery.func9"} 2026-08-01T05:50:41Z ERROR Reconciler error {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "1d8b8b5a-8fbe-435f-b1f9-acf43017c204", "error": "provisioning failed: failure deploying resource /torch-distributed: apply failed trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime: unable to patch trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime torch-distributed: Internal error occurred: failed calling webhook \"validator.clustertrainingruntime.trainer.kubeflow.org\": failed to call webhook: Post \"https://kubeflow-trainer-controller-manager.opendatahub.svc:443/validate-trainer-kubeflow-org-v1alpha1-clustertrainingruntime?timeout=10s\": no endpoints available for service \"kubeflow-trainer-controller-manager\""} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313 2026-08-01T05:51:01Z INFO reconcile {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "cc786467-0a7c-4c35-ad83-1b3097e7c97e"} 2026-08-01T05:51:01Z INFO apply {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "cc786467-0a7c-4c35-ad83-1b3097e7c97e"} 2026-08-01T05:51:01Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "cc786467-0a7c-4c35-ad83-1b3097e7c97e", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).checkDependencies-fm"} 2026-08-01T05:51:01Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "cc786467-0a7c-4c35-ad83-1b3097e7c97e", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).ensureNamespace-fm"} 2026-08-01T05:51:01Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "cc786467-0a7c-4c35-ad83-1b3097e7c97e", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).updateReleases-fm"} 2026-08-01T05:51:01Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "cc786467-0a7c-4c35-ad83-1b3097e7c97e", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm"} 2026/08/01 05:51:02 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T05:51:02Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered manifests {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "cc786467-0a7c-4c35-ad83-1b3097e7c97e", "count": 31} 2026/08/01 05:51:02 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T05:51:02Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered runtimes {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "cc786467-0a7c-4c35-ad83-1b3097e7c97e", "count": 15} 2026/08/01 05:51:02 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T05:51:02Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered imagestreams {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "cc786467-0a7c-4c35-ad83-1b3097e7c97e", "count": 3} 2026-08-01T05:51:02Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "cc786467-0a7c-4c35-ad83-1b3097e7c97e", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.NewReconciler.withImmutableFieldRecovery.func9"} 2026-08-01T05:51:02Z ERROR Reconciler error {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "cc786467-0a7c-4c35-ad83-1b3097e7c97e", "error": "provisioning failed: failure deploying resource /torch-distributed: apply failed trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime: unable to patch trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime torch-distributed: Internal error occurred: failed calling webhook \"validator.clustertrainingruntime.trainer.kubeflow.org\": failed to call webhook: Post \"https://kubeflow-trainer-controller-manager.opendatahub.svc:443/validate-trainer-kubeflow-org-v1alpha1-clustertrainingruntime?timeout=10s\": no endpoints available for service \"kubeflow-trainer-controller-manager\""} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313 2026-08-01T05:51:43Z INFO reconcile {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "4d17c104-391c-4f29-b291-8ed53f253dde"} 2026-08-01T05:51:43Z INFO apply {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "4d17c104-391c-4f29-b291-8ed53f253dde"} 2026-08-01T05:51:43Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "4d17c104-391c-4f29-b291-8ed53f253dde", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).checkDependencies-fm"} 2026-08-01T05:51:43Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "4d17c104-391c-4f29-b291-8ed53f253dde", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).ensureNamespace-fm"} 2026-08-01T05:51:43Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "4d17c104-391c-4f29-b291-8ed53f253dde", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).updateReleases-fm"} 2026-08-01T05:51:43Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "4d17c104-391c-4f29-b291-8ed53f253dde", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm"} 2026/08/01 05:51:44 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T05:51:44Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered manifests {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "4d17c104-391c-4f29-b291-8ed53f253dde", "count": 31} 2026/08/01 05:51:44 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T05:51:44Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered runtimes {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "4d17c104-391c-4f29-b291-8ed53f253dde", "count": 15} 2026/08/01 05:51:44 Warning: Sorting order is set both in 'kustomization.yaml' ('sortOptions') and in a CLI flag ('--reorder'). Using the kustomization file over the CLI flag. 2026-08-01T05:51:44Z INFO action.github.com/opendatahub-io/trainer-operator/internal/controller.(*trainerActions).renderManifests-fm Rendered imagestreams {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "4d17c104-391c-4f29-b291-8ed53f253dde", "count": 3} 2026-08-01T05:51:44Z INFO Executing action {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "4d17c104-391c-4f29-b291-8ed53f253dde", "action": "github.com/opendatahub-io/trainer-operator/internal/controller.NewReconciler.withImmutableFieldRecovery.func9"} 2026-08-01T05:51:45Z ERROR Reconciler error {"controller": "trainer", "controllerGroup": "components.platform.opendatahub.io", "controllerKind": "Trainer", "Trainer": {"name":"default-trainer"}, "namespace": "", "name": "default-trainer", "reconcileID": "4d17c104-391c-4f29-b291-8ed53f253dde", "error": "provisioning failed: failure deploying resource /torch-distributed: apply failed trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime: unable to patch trainer.kubeflow.org/v1alpha1, Kind=ClusterTrainingRuntime torch-distributed: Internal error occurred: failed calling webhook \"validator.clustertrainingruntime.trainer.kubeflow.org\": failed to call webhook: Post \"https://kubeflow-trainer-controller-manager.opendatahub.svc:443/validate-trainer-kubeflow-org-v1alpha1-clustertrainingruntime?timeout=10s\": no endpoints available for service \"kubeflow-trainer-controller-manager\""} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313 client.go:143: Kubernetes events: 0001-01-01T00:00:00Z Normal Scheduled trainer-operator-controller-manager-5d969fb48f-bcpf7 Successfully assigned trainer-operator-system/trainer-operator-controller-manager-5d969fb48f-bcpf7 to ip-10-0-129-202.ec2.internal 2026-08-01T05:49:06Z Normal AddedInterface trainer-operator-controller-manager-5d969fb48f-bcpf7 Add eth0 [10.132.0.30/23] from ovn-kubernetes 2026-08-01T05:49:06Z Normal Pulling trainer-operator-controller-manager-5d969fb48f-bcpf7 Pulling image "quay.io/opendatahub/odh-trainer-operator@sha256:897b44b96939e8315e043cbf6d7a0a3079ed9d283b5e9abec21080c23971cf13" 2026-08-01T05:49:06Z Normal SuccessfulCreate trainer-operator-controller-manager-5d969fb48f Created pod: trainer-operator-controller-manager-5d969fb48f-bcpf7 2026-08-01T05:49:06Z Normal ScalingReplicaSet trainer-operator-controller-manager Scaled up replica set trainer-operator-controller-manager-5d969fb48f from 0 to 1 2026-08-01T05:49:09Z Normal Pulled trainer-operator-controller-manager-5d969fb48f-bcpf7 Successfully pulled image "quay.io/opendatahub/odh-trainer-operator@sha256:897b44b96939e8315e043cbf6d7a0a3079ed9d283b5e9abec21080c23971cf13" in 2.475s (2.475s including waiting). Image size: 197968060 bytes. 2026-08-01T05:49:09Z Normal Created trainer-operator-controller-manager-5d969fb48f-bcpf7 Created container: manager 2026-08-01T05:49:09Z Normal Started trainer-operator-controller-manager-5d969fb48f-bcpf7 Started container manager --- FAIL: TestTrainJobWithClusterTrainingRuntime (121.07s) FAIL FAIL github.com/opendatahub-io/trainer-operator/test/e2e/ocp 121.124s FAIL make: *** [Makefile:151: test-e2e-ocp] Error 1