<?xml version="1.0" encoding="UTF-8"?>
  <testsuites tests="3" disabled="0" errors="0" failures="2" time="1200.927416053">
      <testsuite name="Kubeflow Trainer E2E Suite" package="/workspace/source/src_code/test/e2e" tests="3" disabled="0" skipped="0" errors="0" failures="2" time="1200.927416053" timestamp="2026-10-23T03:38:04">
          <properties>
              <property name="SuiteSucceeded" value="false"></property>
              <property name="SuiteHasProgrammaticFocus" value="false"></property>
              <property name="SpecialSuiteFailureReason" value=""></property>
              <property name="SuiteLabels" value="[]"></property>
              <property name="SuiteSemVerConstraints" value="[]"></property>
              <property name="RandomSeed" value="1792726644"></property>
              <property name="RandomizeAllSpecs" value="false"></property>
              <property name="LabelFilter" value=""></property>
              <property name="SemVerFilter" value=""></property>
              <property name="FocusStrings" value=""></property>
              <property name="SkipStrings" value=""></property>
              <property name="FocusFiles" value=""></property>
              <property name="SkipFiles" value=""></property>
              <property name="FailOnPending" value="false"></property>
              <property name="FailOnEmpty" value="false"></property>
              <property name="FailFast" value="false"></property>
              <property name="FlakeAttempts" value="0"></property>
              <property name="DryRun" value="false"></property>
              <property name="ParallelTotal" value="1"></property>
              <property name="OutputInterceptorMode" value=""></property>
          </properties>
          <testcase name="[BeforeSuite]" classname="Kubeflow Trainer E2E Suite" status="passed" time="0.000931642">
              <system-err>&gt; Enter [BeforeSuite] TOP-LEVEL - /workspace/source/src_code/test/e2e/suite_test.go:42 @ 10/23/26 03:38:04.675&#xA;&lt; Exit [BeforeSuite] TOP-LEVEL - /workspace/source/src_code/test/e2e/suite_test.go:42 @ 10/23/26 03:38:04.676 (1ms)&#xA;</system-err>
          </testcase>
          <testcase name="[It] TrainJob e2e when Creating TrainJob to perform the PyTorch workload should create TrainJob with PyTorch runtime reference" classname="Kubeflow Trainer E2E Suite" status="failed" time="600.803470429">
              <failure message="Timed out after 600.001s.&#xA;The function passed to Eventually failed at /workspace/source/src_code/test/e2e/e2e_test.go:84 with:&#xA;Expected object to be comparable, diff:   []v1.Condition(&#xA;- &#x9;nil,&#xA;+ &#x9;{&#xA;+ &#x9;&#x9;{&#xA;+ &#x9;&#x9;&#x9;Type:    &#34;Complete&#34;,&#xA;+ &#x9;&#x9;&#x9;Status:  &#34;True&#34;,&#xA;+ &#x9;&#x9;&#x9;Reason:  &#34;AllJobsCompleted&#34;,&#xA;+ &#x9;&#x9;&#x9;Message: &#34;jobset completed successfully&#34;,&#xA;+ &#x9;&#x9;},&#xA;+ &#x9;},&#xA;  )&#xA;" type="failed">[FAILED] Timed out after 600.001s.&#xA;The function passed to Eventually failed at /workspace/source/src_code/test/e2e/e2e_test.go:84 with:&#xA;Expected object to be comparable, diff:   []v1.Condition(&#xA;- &#x9;nil,&#xA;+ &#x9;{&#xA;+ &#x9;&#x9;{&#xA;+ &#x9;&#x9;&#x9;Type:    &#34;Complete&#34;,&#xA;+ &#x9;&#x9;&#x9;Status:  &#34;True&#34;,&#xA;+ &#x9;&#x9;&#x9;Reason:  &#34;AllJobsCompleted&#34;,&#xA;+ &#x9;&#x9;&#x9;Message: &#34;jobset completed successfully&#34;,&#xA;+ &#x9;&#x9;},&#xA;+ &#x9;},&#xA;  )&#xA;In [It] at: /workspace/source/src_code/test/e2e/e2e_test.go:102 @ 10/23/26 03:48:05.46&#xA;</failure>
              <system-err>&gt; Enter [BeforeEach] TrainJob e2e - /workspace/source/src_code/test/e2e/e2e_test.go:28 @ 10/23/26 03:38:04.676&#xA;&lt; Exit [BeforeEach] TrainJob e2e - /workspace/source/src_code/test/e2e/e2e_test.go:28 @ 10/23/26 03:38:04.835 (159ms)&#xA;&gt; Enter [It] should create TrainJob with PyTorch runtime reference - /workspace/source/src_code/test/e2e/e2e_test.go:51 @ 10/23/26 03:38:04.835&#xA;STEP: Create a TrainJob with torch-distributed runtime reference - /workspace/source/src_code/test/e2e/e2e_test.go:57 @ 10/23/26 03:38:04.835&#xA;END STEP: Create a TrainJob with torch-distributed runtime reference - /workspace/source/src_code/test/e2e/e2e_test.go:57 @ 10/23/26 03:38:04.91 (75ms)&#xA;STEP: Wait for TrainJob jobs to become active - /workspace/source/src_code/test/e2e/e2e_test.go:62 @ 10/23/26 03:38:04.91&#xA;END STEP: Wait for TrainJob jobs to become active - /workspace/source/src_code/test/e2e/e2e_test.go:62 @ 10/23/26 03:38:05.459 (549ms)&#xA;STEP: Wait for TrainJob to be in Succeeded status with all jobs succeeded - /workspace/source/src_code/test/e2e/e2e_test.go:80 @ 10/23/26 03:38:05.459&#xA;END STEP: Wait for TrainJob to be in Succeeded status with all jobs succeeded - /workspace/source/src_code/test/e2e/e2e_test.go:80 @ 10/23/26 03:48:05.46 (10m0.001s)&#xA;[FAILED] Timed out after 600.001s.&#xA;The function passed to Eventually failed at /workspace/source/src_code/test/e2e/e2e_test.go:84 with:&#xA;Expected object to be comparable, diff:   []v1.Condition(&#xA;- &#x9;nil,&#xA;+ &#x9;{&#xA;+ &#x9;&#x9;{&#xA;+ &#x9;&#x9;&#x9;Type:    &#34;Complete&#34;,&#xA;+ &#x9;&#x9;&#x9;Status:  &#34;True&#34;,&#xA;+ &#x9;&#x9;&#x9;Reason:  &#34;AllJobsCompleted&#34;,&#xA;+ &#x9;&#x9;&#x9;Message: &#34;jobset completed successfully&#34;,&#xA;+ &#x9;&#x9;},&#xA;+ &#x9;},&#xA;  )&#xA;In [It] at: /workspace/source/src_code/test/e2e/e2e_test.go:102 @ 10/23/26 03:48:05.46&#xA;&lt; Exit [It] should create TrainJob with PyTorch runtime reference - /workspace/source/src_code/test/e2e/e2e_test.go:51 @ 10/23/26 03:48:05.461 (10m0.626s)&#xA;&gt; Enter [AfterEach] TrainJob e2e - /workspace/source/src_code/test/e2e/e2e_test.go:43 @ 10/23/26 03:48:05.461&#xA;&lt; Exit [AfterEach] TrainJob e2e - /workspace/source/src_code/test/e2e/e2e_test.go:43 @ 10/23/26 03:48:05.479 (19ms)&#xA;</system-err>
          </testcase>
          <testcase name="[It] TrainJob e2e when Creating TrainJob to perform OpenMPI workload should create TrainJob with DeepSpeed runtime reference" classname="Kubeflow Trainer E2E Suite" status="failed" time="600.122628212">
              <failure message="Timed out after 600.001s.&#xA;The function passed to Eventually failed at /workspace/source/src_code/test/e2e/e2e_test.go:124 with:&#xA;Expected object to be comparable, diff:   []v1alpha1.JobStatus(Inverse(cmpopts.SortSlices, []v1alpha1.JobStatus{&#xA;  &#x9;{&#xA;  &#x9;&#x9;... // 2 identical fields&#xA;  &#x9;&#x9;Succeeded: &amp;0,&#xA;  &#x9;&#x9;Failed:    &amp;0,&#xA;- &#x9;&#x9;Active:    &amp;0,&#xA;+ &#x9;&#x9;Active:    &amp;1,&#xA;  &#x9;&#x9;Suspended: &amp;0,&#xA;  &#x9;},&#xA;  &#x9;{&#xA;  &#x9;&#x9;... // 2 identical fields&#xA;  &#x9;&#x9;Succeeded: &amp;0,&#xA;  &#x9;&#x9;Failed:    &amp;0,&#xA;- &#x9;&#x9;Active:    &amp;0,&#xA;+ &#x9;&#x9;Active:    &amp;1,&#xA;  &#x9;&#x9;Suspended: &amp;0,&#xA;  &#x9;},&#xA;  }))&#xA;" type="failed">[FAILED] Timed out after 600.001s.&#xA;The function passed to Eventually failed at /workspace/source/src_code/test/e2e/e2e_test.go:124 with:&#xA;Expected object to be comparable, diff:   []v1alpha1.JobStatus(Inverse(cmpopts.SortSlices, []v1alpha1.JobStatus{&#xA;  &#x9;{&#xA;  &#x9;&#x9;... // 2 identical fields&#xA;  &#x9;&#x9;Succeeded: &amp;0,&#xA;  &#x9;&#x9;Failed:    &amp;0,&#xA;- &#x9;&#x9;Active:    &amp;0,&#xA;+ &#x9;&#x9;Active:    &amp;1,&#xA;  &#x9;&#x9;Suspended: &amp;0,&#xA;  &#x9;},&#xA;  &#x9;{&#xA;  &#x9;&#x9;... // 2 identical fields&#xA;  &#x9;&#x9;Succeeded: &amp;0,&#xA;  &#x9;&#x9;Failed:    &amp;0,&#xA;- &#x9;&#x9;Active:    &amp;0,&#xA;+ &#x9;&#x9;Active:    &amp;1,&#xA;  &#x9;&#x9;Suspended: &amp;0,&#xA;  &#x9;},&#xA;  }))&#xA;In [It] at: /workspace/source/src_code/test/e2e/e2e_test.go:142 @ 10/23/26 03:58:05.581&#xA;</failure>
              <system-err>&gt; Enter [BeforeEach] TrainJob e2e - /workspace/source/src_code/test/e2e/e2e_test.go:28 @ 10/23/26 03:48:05.48&#xA;&lt; Exit [BeforeEach] TrainJob e2e - /workspace/source/src_code/test/e2e/e2e_test.go:28 @ 10/23/26 03:48:05.512 (32ms)&#xA;&gt; Enter [It] should create TrainJob with DeepSpeed runtime reference - /workspace/source/src_code/test/e2e/e2e_test.go:109 @ 10/23/26 03:48:05.512&#xA;STEP: Create a TrainJob with deepspeed-distributed runtime reference - /workspace/source/src_code/test/e2e/e2e_test.go:115 @ 10/23/26 03:48:05.512&#xA;END STEP: Create a TrainJob with deepspeed-distributed runtime reference - /workspace/source/src_code/test/e2e/e2e_test.go:115 @ 10/23/26 03:48:05.58 (68ms)&#xA;STEP: Wait for TrainJob jobs to become active - /workspace/source/src_code/test/e2e/e2e_test.go:120 @ 10/23/26 03:48:05.58&#xA;END STEP: Wait for TrainJob jobs to become active - /workspace/source/src_code/test/e2e/e2e_test.go:120 @ 10/23/26 03:58:05.581 (10m0.001s)&#xA;[FAILED] Timed out after 600.001s.&#xA;The function passed to Eventually failed at /workspace/source/src_code/test/e2e/e2e_test.go:124 with:&#xA;Expected object to be comparable, diff:   []v1alpha1.JobStatus(Inverse(cmpopts.SortSlices, []v1alpha1.JobStatus{&#xA;  &#x9;{&#xA;  &#x9;&#x9;... // 2 identical fields&#xA;  &#x9;&#x9;Succeeded: &amp;0,&#xA;  &#x9;&#x9;Failed:    &amp;0,&#xA;- &#x9;&#x9;Active:    &amp;0,&#xA;+ &#x9;&#x9;Active:    &amp;1,&#xA;  &#x9;&#x9;Suspended: &amp;0,&#xA;  &#x9;},&#xA;  &#x9;{&#xA;  &#x9;&#x9;... // 2 identical fields&#xA;  &#x9;&#x9;Succeeded: &amp;0,&#xA;  &#x9;&#x9;Failed:    &amp;0,&#xA;- &#x9;&#x9;Active:    &amp;0,&#xA;+ &#x9;&#x9;Active:    &amp;1,&#xA;  &#x9;&#x9;Suspended: &amp;0,&#xA;  &#x9;},&#xA;  }))&#xA;In [It] at: /workspace/source/src_code/test/e2e/e2e_test.go:142 @ 10/23/26 03:58:05.581&#xA;&lt; Exit [It] should create TrainJob with DeepSpeed runtime reference - /workspace/source/src_code/test/e2e/e2e_test.go:109 @ 10/23/26 03:58:05.581 (10m0.07s)&#xA;&gt; Enter [AfterEach] TrainJob e2e - /workspace/source/src_code/test/e2e/e2e_test.go:43 @ 10/23/26 03:58:05.581&#xA;&lt; Exit [AfterEach] TrainJob e2e - /workspace/source/src_code/test/e2e/e2e_test.go:43 @ 10/23/26 03:58:05.602 (21ms)&#xA;</system-err>
          </testcase>
      </testsuite>
      <testsuite name="" package="/workspace/source/src_code/test/e2e/rhai" tests="0" disabled="0" skipped="0" errors="0" failures="0" time="0" timestamp="0001-01-01T00:00:00">
          <properties>
              <property name="SuiteSucceeded" value="false"></property>
              <property name="SuiteHasProgrammaticFocus" value="false"></property>
              <property name="SpecialSuiteFailureReason" value="Suite did not run because prior suites failed and --keep-going is not set"></property>
              <property name="SuiteLabels" value="[]"></property>
              <property name="SuiteSemVerConstraints" value="[]"></property>
              <property name="RandomSeed" value="1792726644"></property>
              <property name="RandomizeAllSpecs" value="false"></property>
              <property name="LabelFilter" value=""></property>
              <property name="SemVerFilter" value=""></property>
              <property name="FocusStrings" value=""></property>
              <property name="SkipStrings" value=""></property>
              <property name="FocusFiles" value=""></property>
              <property name="SkipFiles" value=""></property>
              <property name="FailOnPending" value="false"></property>
              <property name="FailOnEmpty" value="false"></property>
              <property name="FailFast" value="false"></property>
              <property name="FlakeAttempts" value="0"></property>
              <property name="DryRun" value="false"></property>
              <property name="ParallelTotal" value="1"></property>
              <property name="OutputInterceptorMode" value=""></property>
          </properties>
      </testsuite>
  </testsuites>