go: downloading github.com/kubeflow/trainer/v2 v2.1.0
go: downloading k8s.io/api v0.34.2
go: downloading k8s.io/apimachinery v0.34.2
go: downloading k8s.io/client-go v0.34.2
go: downloading sigs.k8s.io/kueue v0.15.7
go: downloading github.com/kubeflow/training-operator v1.7.0
go: downloading github.com/openshift/api v0.0.0-20251124165233-999c45c0835a
go: downloading github.com/openshift/client-go v0.0.0-20251015124057-db0dee36e235
go: downloading github.com/openshift/kueue-operator v0.0.0-20251202204851-958c48004dad
go: downloading github.com/operator-framework/api v0.36.0
go: downloading github.com/operator-framework/operator-lifecycle-manager v0.38.0
go: downloading github.com/prometheus/common v0.67.2
go: downloading github.com/ray-project/kuberay/ray-operator v1.5.1
go: downloading github.com/minio/minio-go/v7 v7.0.98
go: downloading k8s.io/utils v0.0.0-20251002143259-bc988d571ff4
go: downloading k8s.io/kube-openapi v0.0.0-20251125145642-4e65d59e963e
go: downloading sigs.k8s.io/controller-runtime v0.22.4
go: downloading github.com/sirupsen/logrus v1.9.3
go: downloading sigs.k8s.io/structured-merge-diff/v6 v6.3.1
go: downloading github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f
go: downloading go.yaml.in/yaml/v2 v2.4.3
go: downloading golang.org/x/net v0.48.0
go: downloading golang.org/x/oauth2 v0.33.0
go: downloading golang.org/x/term v0.38.0
go: downloading github.com/spf13/pflag v1.0.10
go: downloading sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730
go: downloading github.com/go-ini/ini v1.67.0
go: downloading github.com/dustin/go-humanize v1.0.1
go: downloading github.com/klauspost/compress v1.18.2
go: downloading github.com/klauspost/crc32 v1.3.0
go: downloading github.com/minio/crc64nvme v1.1.1
go: downloading github.com/minio/md5-simd v1.1.2
go: downloading github.com/google/gnostic-models v0.7.1
go: downloading google.golang.org/protobuf v1.36.10
go: downloading golang.org/x/time v0.14.0
go: downloading golang.org/x/sys v0.39.0
go: downloading gopkg.in/evanphx/json-patch.v4 v4.13.0
go: downloading github.com/emicklei/go-restful/v3 v3.13.0
go: downloading github.com/go-openapi/jsonreference v0.21.3
go: downloading github.com/go-openapi/swag v0.25.4
go: downloading github.com/blang/semver/v4 v4.0.0
go: downloading github.com/jpillora/backoff v1.0.0
go: downloading golang.org/x/text v0.32.0
go: downloading github.com/klauspost/cpuid/v2 v2.2.11
go: downloading github.com/rs/xid v1.6.0
go: downloading golang.org/x/crypto v0.46.0
go: downloading github.com/tinylib/msgp v1.6.1
go: downloading github.com/go-openapi/jsonpointer v0.22.3
go: downloading github.com/go-openapi/swag/cmdutils v0.25.4
go: downloading github.com/go-openapi/swag/conv v0.25.4
go: downloading github.com/go-openapi/swag/fileutils v0.25.4
go: downloading github.com/go-openapi/swag/jsonname v0.25.4
go: downloading github.com/go-openapi/swag/jsonutils v0.25.4
go: downloading github.com/go-openapi/swag/loading v0.25.4
go: downloading github.com/go-openapi/swag/mangling v0.25.4
go: downloading github.com/go-openapi/swag/netutils v0.25.4
go: downloading github.com/go-openapi/swag/stringutils v0.25.4
go: downloading github.com/go-openapi/swag/typeutils v0.25.4
go: downloading github.com/go-openapi/swag/yamlutils v0.25.4
go: downloading github.com/prometheus/procfs v0.17.0
go: downloading github.com/philhofer/fwd v1.2.0
Warning: Failed to get DSC: the server could not find the requested resource
Initial Kueue managementState: 
=== RUN   TestOpenMPICudaClusterTrainingRuntime
    cluster_training_runtimes_test.go:120: Skip due to issue RHOAIENG-61966
--- SKIP: TestOpenMPICudaClusterTrainingRuntime (0.00s)
=== RUN   TestDefaultTrainingHubRuntimesMatchDefaultClusterRuntimes
    cluster_training_runtimes_test.go:171: CTR "training-hub-rocm" matches DefaultClusterTrainingRuntime "torch-distributed-rocm"
    cluster_training_runtimes_test.go:171: CTR "training-hub-th09-rocm714-torch211-py312" matches DefaultClusterTrainingRuntime "torch-distributed-rocm"
    cluster_training_runtimes_test.go:171: CTR "torch-distributed-cuda130-torch211-py312" matches DefaultClusterTrainingRuntime "torch-distributed"
    cluster_training_runtimes_test.go:171: CTR "torch-distributed-cpu-torch211-py312" matches DefaultClusterTrainingRuntime "torch-distributed-cpu"
    cluster_training_runtimes_test.go:171: CTR "training-hub" matches DefaultClusterTrainingRuntime "torch-distributed"
    cluster_training_runtimes_test.go:171: CTR "training-hub-cpu" matches DefaultClusterTrainingRuntime "torch-distributed-cpu"
    cluster_training_runtimes_test.go:171: CTR "training-hub-th09-cuda130-torch211-py312" matches DefaultClusterTrainingRuntime "torch-distributed"
    cluster_training_runtimes_test.go:171: CTR "training-hub-th09-cpu-torch211-py312" matches DefaultClusterTrainingRuntime "torch-distributed-cpu"
    cluster_training_runtimes_test.go:171: CTR "torch-distributed-rocm714-torch211-py312" matches DefaultClusterTrainingRuntime "torch-distributed-rocm"
    cluster_training_runtimes_test.go:174: All CTRs match their DefaultClusterTrainingRuntime counterparts!
--- PASS: TestDefaultTrainingHubRuntimesMatchDefaultClusterRuntimes (0.03s)
=== RUN   TestRunTrainJobWithDefaultClusterTrainingRuntimes
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'Tier1'
--- SKIP: TestRunTrainJobWithDefaultClusterTrainingRuntimes (0.00s)
=== RUN   TestJobSetWorkflow
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'Tier1'
--- SKIP: TestJobSetWorkflow (0.00s)
=== RUN   TestFailedJobSetWorkflow
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'Tier2'
--- SKIP: TestFailedJobSetWorkflow (0.00s)
=== RUN   TestKubeflowSdk
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'Tier1'
--- SKIP: TestKubeflowSdk (0.00s)
=== RUN   TestKubeflowSdkKueueIntegration
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'Tier1'
--- SKIP: TestKubeflowSdkKueueIntegration (0.00s)
=== RUN   TestKubeflowSdkOpenMPICudaKueueIntegration
    kubeflow_sdk_test.go:40: Skip due to issue RHOAIENG-61966
--- SKIP: TestKubeflowSdkOpenMPICudaKueueIntegration (0.00s)
=== RUN   TestSftTrainingHubSingleNodeSingleGPU
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-CUDA'
--- SKIP: TestSftTrainingHubSingleNodeSingleGPU (0.00s)
=== RUN   TestOsftTrainingHubSingleNodeSingleGPU
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-CUDA'
--- SKIP: TestOsftTrainingHubSingleNodeSingleGPU (0.00s)
=== RUN   TestLoraTrainingHubSingleNodeSingleGPU
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-CUDA'
--- SKIP: TestLoraTrainingHubSingleNodeSingleGPU (0.00s)
=== RUN   TestOsftTrainingHubMultiNodeMultiGPU
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-CUDA'
--- SKIP: TestOsftTrainingHubMultiNodeMultiGPU (0.00s)
=== RUN   TestLoraTrainingHubMultiNodeMultiGPU
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-CUDA'
--- SKIP: TestLoraTrainingHubMultiNodeMultiGPU (0.00s)
=== RUN   TestSftTrainingHubMultiNodeMultiGPU
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-CUDA'
--- SKIP: TestSftTrainingHubMultiNodeMultiGPU (0.00s)
=== RUN   TestRhaiTrainingProgressionCPU
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'Tier2'
--- SKIP: TestRhaiTrainingProgressionCPU (0.00s)
=== RUN   TestRhaiJitCheckpointingCPU
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'Tier2'
--- SKIP: TestRhaiJitCheckpointingCPU (0.00s)
=== RUN   TestRhaiFeaturesCPU
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'Tier2'
--- SKIP: TestRhaiFeaturesCPU (0.00s)
=== RUN   TestRhaiTrainingProgressionCuda
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-CUDA'
--- SKIP: TestRhaiTrainingProgressionCuda (0.00s)
=== RUN   TestRhaiJitCheckpointingCuda
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-CUDA'
--- SKIP: TestRhaiJitCheckpointingCuda (0.00s)
=== RUN   TestRhaiFeaturesCuda
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-CUDA'
--- SKIP: TestRhaiFeaturesCuda (0.00s)
=== RUN   TestRhaiTrainingProgressionRocm
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-ROCm'
--- SKIP: TestRhaiTrainingProgressionRocm (0.00s)
=== RUN   TestRhaiJitCheckpointingRocm
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-ROCm'
--- SKIP: TestRhaiJitCheckpointingRocm (0.00s)
=== RUN   TestRhaiFeaturesRocm
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-ROCm'
--- SKIP: TestRhaiFeaturesRocm (0.00s)
=== RUN   TestRhaiTrainingProgressionMultiGpuCuda
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-CUDA'
--- SKIP: TestRhaiTrainingProgressionMultiGpuCuda (0.00s)
=== RUN   TestRhaiJitCheckpointingMultiGpuCuda
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-CUDA'
--- SKIP: TestRhaiJitCheckpointingMultiGpuCuda (0.00s)
=== RUN   TestRhaiFeaturesMultiGpuCuda
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-CUDA'
--- SKIP: TestRhaiFeaturesMultiGpuCuda (0.00s)
=== RUN   TestRhaiTrainingProgressionMultiGpuRocm
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-ROCm'
--- SKIP: TestRhaiTrainingProgressionMultiGpuRocm (0.00s)
=== RUN   TestRhaiJitCheckpointingMultiGpuRocm
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-ROCm'
--- SKIP: TestRhaiJitCheckpointingMultiGpuRocm (0.00s)
=== RUN   TestRhaiFeaturesMultiGpuRocm
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-ROCm'
--- SKIP: TestRhaiFeaturesMultiGpuRocm (0.00s)
=== RUN   TestTrainingFailureScenarios
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-CUDA'
--- SKIP: TestTrainingFailureScenarios (0.00s)
=== RUN   TestTorchrunTrainingFailure
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-CUDA'
--- SKIP: TestTorchrunTrainingFailure (0.00s)
=== RUN   TestRhaiS3CheckpointingCPU
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'Tier2'
--- SKIP: TestRhaiS3CheckpointingCPU (0.00s)
=== RUN   TestRhaiS3FsdpFullStateCheckpointingCuda
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-CUDA'
--- SKIP: TestRhaiS3FsdpFullStateCheckpointingCuda (0.00s)
=== RUN   TestRhaiS3FsdpFullStateCheckpointingMultiProcessCuda
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-CUDA'
--- SKIP: TestRhaiS3FsdpFullStateCheckpointingMultiProcessCuda (0.00s)
=== RUN   TestRhaiS3FsdpSharedStateCheckpointingCuda
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-CUDA'
--- SKIP: TestRhaiS3FsdpSharedStateCheckpointingCuda (0.00s)
=== RUN   TestRhaiS3FsdpSharedStateCheckpointingMultiGpuCuda
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-CUDA'
--- SKIP: TestRhaiS3FsdpSharedStateCheckpointingMultiGpuCuda (0.00s)
=== RUN   TestRhaiS3DeepspeedStage0CheckpointingCuda
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-CUDA'
--- SKIP: TestRhaiS3DeepspeedStage0CheckpointingCuda (0.00s)
=== RUN   TestRhaiS3DeepspeedStage0CheckpointingMultiGpuCuda
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-CUDA'
--- SKIP: TestRhaiS3DeepspeedStage0CheckpointingMultiGpuCuda (0.00s)
=== RUN   TestPyTorchDDPMultiNodeMultiCPUWithTorchCuda28
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'Tier2'
--- SKIP: TestPyTorchDDPMultiNodeMultiCPUWithTorchCuda28 (0.00s)
=== RUN   TestPyTorchDDPSingleNodeSingleGPUWithTorchCuda
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-CUDA'
--- SKIP: TestPyTorchDDPSingleNodeSingleGPUWithTorchCuda (0.00s)
=== RUN   TestPyTorchDDPSingleNodeMultiGPUWithTorchCuda
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-CUDA'
--- SKIP: TestPyTorchDDPSingleNodeMultiGPUWithTorchCuda (0.00s)
=== RUN   TestPyTorchDDPMultiNodeSingleGPUWithTorchCuda
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-CUDA'
--- SKIP: TestPyTorchDDPMultiNodeSingleGPUWithTorchCuda (0.00s)
=== RUN   TestPyTorchDDPMultiNodeMultiGPUWithTorchCuda
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-CUDA'
--- SKIP: TestPyTorchDDPMultiNodeMultiGPUWithTorchCuda (0.00s)
=== RUN   TestPyTorchDDPSingleNodeSingleGPUWithTorchRocm
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-ROCm'
--- SKIP: TestPyTorchDDPSingleNodeSingleGPUWithTorchRocm (0.00s)
=== RUN   TestPyTorchDDPSingleNodeMultiGPUWithTorchRocm
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-ROCm'
--- SKIP: TestPyTorchDDPSingleNodeMultiGPUWithTorchRocm (0.00s)
=== RUN   TestPyTorchDDPMultiNodeSingleGPUWithTorchRocm
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-ROCm'
--- SKIP: TestPyTorchDDPMultiNodeSingleGPUWithTorchRocm (0.00s)
=== RUN   TestPyTorchDDPMultiNodeMultiGPUWithTorchRocm
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-ROCm'
--- SKIP: TestPyTorchDDPMultiNodeMultiGPUWithTorchRocm (0.00s)
=== RUN   TestKueueWorkloadPreemptionSuspendsTrainJob
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'Tier1'
--- SKIP: TestKueueWorkloadPreemptionSuspendsTrainJob (0.00s)
=== RUN   TestKueueWorkloadInadmissibleWithNonExistentLocalQueue
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'Tier2'
--- SKIP: TestKueueWorkloadInadmissibleWithNonExistentLocalQueue (0.00s)
=== RUN   TestKueueSchedulingReachesTrainJobPods
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'Tier1'
--- SKIP: TestKueueSchedulingReachesTrainJobPods (0.00s)
=== RUN   TestSetupUpgradeTrainJob
    trainer_kueue_upgrade_training_test.go:82: Skip due to issue RHOAIENG-48867
--- SKIP: TestSetupUpgradeTrainJob (0.00s)
=== RUN   TestRunUpgradeTrainJob
    trainer_kueue_upgrade_training_test.go:157: Skip due to issue RHOAIENG-48867
--- SKIP: TestRunUpgradeTrainJob (0.00s)
=== RUN   TestSetupSpecificRuntimeUpgradeTrainJob
    trainer_kueue_upgrade_training_test.go:204: Skip due to issue RHOAIENG-48867
--- SKIP: TestSetupSpecificRuntimeUpgradeTrainJob (0.00s)
=== RUN   TestRunSpecificRuntimeUpgradeTrainJob
    trainer_kueue_upgrade_training_test.go:294: Skip due to issue RHOAIENG-48867
--- SKIP: TestRunSpecificRuntimeUpgradeTrainJob (0.00s)
=== RUN   TestSetupCustomRuntimeUpgradeTrainJob
    trainer_kueue_upgrade_training_test.go:372: Skip due to issue RHOAIENG-87834
--- SKIP: TestSetupCustomRuntimeUpgradeTrainJob (0.00s)
=== RUN   TestRunCustomRuntimeUpgradeTrainJob
    trainer_kueue_upgrade_training_test.go:458: Skip due to issue RHOAIENG-87834
--- SKIP: TestRunCustomRuntimeUpgradeTrainJob (0.00s)
=== RUN   TestMultiNodeOpenMPITrainJob
    trainer_mpi_test.go:35: Skip due to issue RHOAIENG-61966
--- SKIP: TestMultiNodeOpenMPITrainJob (0.00s)
=== RUN   TestOpenMPICudaTrainJobKueueIntegration
    trainer_openmpi_kueue_integration_test.go:36: Skip due to issue RHOAIENG-61966
--- SKIP: TestOpenMPICudaTrainJobKueueIntegration (0.00s)
=== RUN   TestOpenMPICudaTrainJobKueueWorkloadDeactivateReactivate
    trainer_openmpi_kueue_integration_test.go:138: Skip due to issue RHOAIENG-61966
--- SKIP: TestOpenMPICudaTrainJobKueueWorkloadDeactivateReactivate (0.00s)
=== RUN   TestSftStockTrlSingleNodeSingleGPU
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-CUDA'
--- SKIP: TestSftStockTrlSingleNodeSingleGPU (0.00s)
=== RUN   TestSftStockTrlSingleNodeSingleGPUWithTorchRocm
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-ROCm'
--- SKIP: TestSftStockTrlSingleNodeSingleGPUWithTorchRocm (0.00s)
=== RUN   TestSetupTrainingRuntime
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'Pre-Upgrade'
--- SKIP: TestSetupTrainingRuntime (0.00s)
=== RUN   TestVerifyTrainingRuntime
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'Post-Upgrade'
--- SKIP: TestVerifyTrainingRuntime (0.00s)
=== RUN   TestSetupSleepTrainJob
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'Pre-Upgrade'
--- SKIP: TestSetupSleepTrainJob (0.00s)
=== RUN   TestVerifySleepTrainJob
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'Post-Upgrade'
--- SKIP: TestVerifySleepTrainJob (0.00s)
PASS
TearDown: Setting kueue managementState to Removed in DataScienceCluster...
TearDown: Failed to set Kueue to Removed: TearDown: failed to set kueue to Removed: the server could not find the requested resource
ok  	github.com/opendatahub-io/distributed-workloads/tests/trainer	0.132s

=== Skipped
=== SKIP: tests/trainer TestOpenMPICudaClusterTrainingRuntime (0.00s)
    cluster_training_runtimes_test.go:120: Skip due to issue RHOAIENG-61966

=== SKIP: tests/trainer TestRunTrainJobWithDefaultClusterTrainingRuntimes (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'Tier1'

=== SKIP: tests/trainer TestJobSetWorkflow (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'Tier1'

=== SKIP: tests/trainer TestFailedJobSetWorkflow (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'Tier2'

=== SKIP: tests/trainer TestKubeflowSdk (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'Tier1'

=== SKIP: tests/trainer TestKubeflowSdkKueueIntegration (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'Tier1'

=== SKIP: tests/trainer TestKubeflowSdkOpenMPICudaKueueIntegration (0.00s)
    kubeflow_sdk_test.go:40: Skip due to issue RHOAIENG-61966

=== SKIP: tests/trainer TestSftTrainingHubSingleNodeSingleGPU (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-CUDA'

=== SKIP: tests/trainer TestOsftTrainingHubSingleNodeSingleGPU (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-CUDA'

=== SKIP: tests/trainer TestLoraTrainingHubSingleNodeSingleGPU (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-CUDA'

=== SKIP: tests/trainer TestOsftTrainingHubMultiNodeMultiGPU (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-CUDA'

=== SKIP: tests/trainer TestLoraTrainingHubMultiNodeMultiGPU (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-CUDA'

=== SKIP: tests/trainer TestSftTrainingHubMultiNodeMultiGPU (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-CUDA'

=== SKIP: tests/trainer TestRhaiTrainingProgressionCPU (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'Tier2'

=== SKIP: tests/trainer TestRhaiJitCheckpointingCPU (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'Tier2'

=== SKIP: tests/trainer TestRhaiFeaturesCPU (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'Tier2'

=== SKIP: tests/trainer TestRhaiTrainingProgressionCuda (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-CUDA'

=== SKIP: tests/trainer TestRhaiJitCheckpointingCuda (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-CUDA'

=== SKIP: tests/trainer TestRhaiFeaturesCuda (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-CUDA'

=== SKIP: tests/trainer TestRhaiTrainingProgressionRocm (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-ROCm'

=== SKIP: tests/trainer TestRhaiJitCheckpointingRocm (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-ROCm'

=== SKIP: tests/trainer TestRhaiFeaturesRocm (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-ROCm'

=== SKIP: tests/trainer TestRhaiTrainingProgressionMultiGpuCuda (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-CUDA'

=== SKIP: tests/trainer TestRhaiJitCheckpointingMultiGpuCuda (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-CUDA'

=== SKIP: tests/trainer TestRhaiFeaturesMultiGpuCuda (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-CUDA'

=== SKIP: tests/trainer TestRhaiTrainingProgressionMultiGpuRocm (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-ROCm'

=== SKIP: tests/trainer TestRhaiJitCheckpointingMultiGpuRocm (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-ROCm'

=== SKIP: tests/trainer TestRhaiFeaturesMultiGpuRocm (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-ROCm'

=== SKIP: tests/trainer TestTrainingFailureScenarios (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-CUDA'

=== SKIP: tests/trainer TestTorchrunTrainingFailure (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-CUDA'

=== SKIP: tests/trainer TestRhaiS3CheckpointingCPU (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'Tier2'

=== SKIP: tests/trainer TestRhaiS3FsdpFullStateCheckpointingCuda (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-CUDA'

=== SKIP: tests/trainer TestRhaiS3FsdpFullStateCheckpointingMultiProcessCuda (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-CUDA'

=== SKIP: tests/trainer TestRhaiS3FsdpSharedStateCheckpointingCuda (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-CUDA'

=== SKIP: tests/trainer TestRhaiS3FsdpSharedStateCheckpointingMultiGpuCuda (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-CUDA'

=== SKIP: tests/trainer TestRhaiS3DeepspeedStage0CheckpointingCuda (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-CUDA'

=== SKIP: tests/trainer TestRhaiS3DeepspeedStage0CheckpointingMultiGpuCuda (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-CUDA'

=== SKIP: tests/trainer TestPyTorchDDPMultiNodeMultiCPUWithTorchCuda28 (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'Tier2'

=== SKIP: tests/trainer TestPyTorchDDPSingleNodeSingleGPUWithTorchCuda (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-CUDA'

=== SKIP: tests/trainer TestPyTorchDDPSingleNodeMultiGPUWithTorchCuda (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-CUDA'

=== SKIP: tests/trainer TestPyTorchDDPMultiNodeSingleGPUWithTorchCuda (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-CUDA'

=== SKIP: tests/trainer TestPyTorchDDPMultiNodeMultiGPUWithTorchCuda (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-CUDA'

=== SKIP: tests/trainer TestPyTorchDDPSingleNodeSingleGPUWithTorchRocm (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-ROCm'

=== SKIP: tests/trainer TestPyTorchDDPSingleNodeMultiGPUWithTorchRocm (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-ROCm'

=== SKIP: tests/trainer TestPyTorchDDPMultiNodeSingleGPUWithTorchRocm (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-ROCm'

=== SKIP: tests/trainer TestPyTorchDDPMultiNodeMultiGPUWithTorchRocm (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-ROCm'

=== SKIP: tests/trainer TestKueueWorkloadPreemptionSuspendsTrainJob (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'Tier1'

=== SKIP: tests/trainer TestKueueWorkloadInadmissibleWithNonExistentLocalQueue (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'Tier2'

=== SKIP: tests/trainer TestKueueSchedulingReachesTrainJobPods (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'Tier1'

=== SKIP: tests/trainer TestSetupUpgradeTrainJob (0.00s)
    trainer_kueue_upgrade_training_test.go:82: Skip due to issue RHOAIENG-48867

=== SKIP: tests/trainer TestRunUpgradeTrainJob (0.00s)
    trainer_kueue_upgrade_training_test.go:157: Skip due to issue RHOAIENG-48867

=== SKIP: tests/trainer TestSetupSpecificRuntimeUpgradeTrainJob (0.00s)
    trainer_kueue_upgrade_training_test.go:204: Skip due to issue RHOAIENG-48867

=== SKIP: tests/trainer TestRunSpecificRuntimeUpgradeTrainJob (0.00s)
    trainer_kueue_upgrade_training_test.go:294: Skip due to issue RHOAIENG-48867

=== SKIP: tests/trainer TestSetupCustomRuntimeUpgradeTrainJob (0.00s)
    trainer_kueue_upgrade_training_test.go:372: Skip due to issue RHOAIENG-87834

=== SKIP: tests/trainer TestRunCustomRuntimeUpgradeTrainJob (0.00s)
    trainer_kueue_upgrade_training_test.go:458: Skip due to issue RHOAIENG-87834

=== SKIP: tests/trainer TestMultiNodeOpenMPITrainJob (0.00s)
    trainer_mpi_test.go:35: Skip due to issue RHOAIENG-61966

=== SKIP: tests/trainer TestOpenMPICudaTrainJobKueueIntegration (0.00s)
    trainer_openmpi_kueue_integration_test.go:36: Skip due to issue RHOAIENG-61966

=== SKIP: tests/trainer TestOpenMPICudaTrainJobKueueWorkloadDeactivateReactivate (0.00s)
    trainer_openmpi_kueue_integration_test.go:138: Skip due to issue RHOAIENG-61966

=== SKIP: tests/trainer TestSftStockTrlSingleNodeSingleGPU (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-CUDA'

=== SKIP: tests/trainer TestSftStockTrlSingleNodeSingleGPUWithTorchRocm (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'KFTO-ROCm'

=== SKIP: tests/trainer TestSetupTrainingRuntime (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'Pre-Upgrade'

=== SKIP: tests/trainer TestVerifyTrainingRuntime (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'Post-Upgrade'

=== SKIP: tests/trainer TestSetupSleepTrainJob (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'Pre-Upgrade'

=== SKIP: tests/trainer TestVerifySleepTrainJob (0.00s)
    test_tag.go:37: Test tier 'Smoke' doesn't match expected tier 'Post-Upgrade'

DONE 65 tests, 64 skipped in 0.132s