<?xml version="1.0" encoding="UTF-8"?>
<testsuites tests="107" failures="1" skipped="35">
	<testsuite name="" tests="107" failures="1" errors="0" id="0" hostname="e2e-cli-nmafd0e77d4aef03db7c4ffc26558d59d8e8368c76fb824ea1b-pod" skipped="35" time="1568.800" timestamp="2026-09-15T08:15:42Z">
		<testcase name="TestOdhOperator" classname="" time="393.210">
			<system-out><![CDATA[2026/09/15 08:09:09 ARTIFACT_DIR not set, skipping periodic metrics collection
2026-09-15T08:09:15Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.5.1"}, "Cluster": {"type":"OpenShift","version":"4.20.12"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests" classname="" time="0.240">
			<system-out><![CDATA[2026-09-15T08:09:15Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.5.1"}, "Cluster": {"type":"OpenShift","version":"4.20.12"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_Datascience_Cluster_CRD" classname="" time="0.030"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_DataScienceCluster_Initialization_CRD" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_FeatureTracker_CRD" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_Ray_CRD" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_ModelRegistry_CRD" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_TrustyAI_CRD" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_Kueue_CRD" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_TrainingOperator_CRD" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_Trainer_CRD" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_DataSciencePipelines_CRD" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_Monitoring_CRD" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_Auth_CRD" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_SparkOperator_CRD" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_Platform_CRD" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/DSCInitialization_and_DataScienceCluster_validation_E2E_Tests" classname="" time="0.500">
			<system-out><![CDATA[2026-09-15T08:09:15Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.5.1"}, "Cluster": {"type":"OpenShift","version":"4.20.12"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/DSCInitialization_and_DataScienceCluster_validation_E2E_Tests/Validate_HardwareProfile_resource" classname="" time="10.230"></testcase>
		<testcase name="TestOdhOperator/DSCInitialization_and_DataScienceCluster_validation_E2E_Tests/Validate_owned_namespaces_exist" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/DSCInitialization_and_DataScienceCluster_validation_E2E_Tests/Validate_default_NetworkPolicy_exist" classname="" time="0.050"></testcase>
		<testcase name="TestOdhOperator/DSCInitialization_and_DataScienceCluster_validation_E2E_Tests/Validate_DSCI_webhook_denies_invalid_PEM_on_update" classname="" time="0.060"></testcase>
		<testcase name="TestOdhOperator/DSCInitialization_and_DataScienceCluster_validation_E2E_Tests/Validate_DSCI_webhook_allows_valid_PEM_on_update" classname="" time="0.160"></testcase>
		<testcase name="TestOdhOperator/DSCInitialization_and_DataScienceCluster_validation_E2E_Tests/Validate_DSCI_webhook_allows_invalid_PEM_when_Removed" classname="" time="0.100"></testcase>
		<testcase name="TestOdhOperator/monitoring" classname="" time="0.210">
			<skipped message="Skipped"><![CDATA[2026-09-15T08:09:26Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.5.1"}, "Cluster": {"type":"OpenShift","version":"4.20.12"}}
    controller_test.go:767: Skipping test: passed tag: Smoke, test tags: [Tier2]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components" classname="" time="373.000"></testcase>
		<testcase name="TestOdhOperator/components/group_1" classname="" time="0.000">
			<system-out><![CDATA[    controller_test.go:767: Skipping tests for components/aigateway
    controller_test.go:767: Skipping tests for components/dashboard
    controller_test.go:767: Skipping tests for components/datasciencepipelines
    controller_test.go:767: Skipping tests for components/feastoperator
    controller_test.go:767: Skipping tests for components/kserve
    controller_test.go:767: Skipping tests for components/mcplifecycleoperator
    controller_test.go:767: Skipping tests for components/ogx
    controller_test.go:767: Skipping tests for components/ray]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/sparkoperator" classname="" time="372.150">
			<system-out><![CDATA[2026-09-15T08:09:26Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.5.1"}, "Cluster": {"type":"OpenShift","version":"4.20.12"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/sparkoperator/Validate_component_enabled" classname="" time="30.530"></testcase>
		<testcase name="TestOdhOperator/components/group_1/sparkoperator/Validate_operands_have_OwnerReferences" classname="" time="0.030"></testcase>
		<testcase name="TestOdhOperator/components/group_1/sparkoperator/Validate_update_operand_resources" classname="" time="41.140"></testcase>
		<testcase name="TestOdhOperator/components/group_1/sparkoperator/Validate_update_operand_resources/deployment_spark-operator-controller" classname="" time="20.050"></testcase>
		<testcase name="TestOdhOperator/components/group_1/sparkoperator/Validate_update_operand_resources/deployment_spark-operator-webhook" classname="" time="20.990"></testcase>
		<testcase name="TestOdhOperator/components/group_1/sparkoperator/Validate_component_releases" classname="" time="0.040"></testcase>
		<testcase name="TestOdhOperator/components/group_1/sparkoperator/Validate_platform_release" classname="" time="0.050"></testcase>
		<testcase name="TestOdhOperator/components/group_1/sparkoperator/Validate_SparkPi_workload_execution" classname="" time="300.190">
			<failure message="Failed"><![CDATA[    helper_test.go:157: Creating SparkApplication spark-pi-dakfqvm5v4ss7nr1bqt0 in namespace redhat-ods-applications
    helper_test.go:157: Waiting for SparkApplication spark-pi-dakfqvm5v4ss7nr1bqt0 to complete
    test_context_test.go:228: 
        Timed out after 300.008s.
        The function passed to Eventually failed at /workspace/tests/e2e/resource_applier_test.go:121 with:
        SparkApplication spark-pi-dakfqvm5v4ss7nr1bqt0 did not complete successfully
        Expected
            <string>: map[apiVersion:sparkoperator.k8s.io/v1beta2 kind:SparkApplication metadata:map[creationTimestamp:2026-09-15T08:10:38Z generation:1 managedFields:[map[apiVersion:sparkoperator.k8s.io/v1beta2 fieldsType:FieldsV1 fieldsV1:map[f:spec:map[.:map[] f:arguments:map[] f:driver:map[.:map[] f:coreLimit:map[] f:cores:map[] f:labels:map[.:map[] f:version:map[]] f:memory:map[] f:securityContext:map[] f:serviceAccount:map[] f:volumeMounts:map[]] f:executor:map[.:map[] f:cores:map[] f:instances:map[] f:labels:map[.:map[] f:version:map[]] f:memory:map[] f:securityContext:map[] f:volumeMounts:map[]] f:image:map[] f:imagePullPolicy:map[] f:mainApplicationFile:map[] f:mainClass:map[] f:mode:map[] f:restartPolicy:map[.:map[] f:type:map[]] f:sparkConf:map[.:map[] f:spark.blockManager.port:map[] f:spark.driver.blockManager.port:map[] f:spark.driver.port:map[] f:spark.port.maxRetries:map[]] f:sparkVersion:map[] f:type:map[] f:volumes:map[]]] manager:e2e-tests operation:Update time:2026-09-15T08:10:38Z] map[apiV
ersion:sparkoperator.k8s.io/v1beta2 fieldsType:FieldsV1 fieldsV1:map[f:status:map[.:map[] f:applicationState:map[.:map[] f:state:map[]] f:driverInfo:map[.:map[] f:podName:map[] f:webUIAddress:map[] f:webUIPort:map[] f:webUIServiceName:map[]] f:executionAttempts:map[] f:lastSubmissionAttemptTime:map[] f:sparkApplicationId:map[] f:submissionAttempts:map[] f:submissionID:map[] f:terminationTime:map[]]] manager:spark-operator operation:Update subresource:status time:2026-09-15T08:10:55Z]] name:spark-pi-dakfqvm5v4ss7nr1bqt0 namespace:redhat-ods-applications resourceVersion:308514910 uid:67adae94-dac2-4a84-9452-65deccd0a57f] spec:map[arguments:[1000] driver:map[coreLimit:1200m cores:1 labels:map[version:4.0.1] memory:512m securityContext:map[] serviceAccount:spark-operator-spark volumeMounts:[map[mountPath:/opt/spark/work-dir name:spark-work-dir]]] executor:map[cores:1 instances:1 labels:map[version:4.0.1] memory:512m securityContext:map[] volumeMounts:[map[mountPath:/opt/spark/work-dir name:spark-work-dir]]] image
:quay.io/opendatahub/data-processing:Spark-v4.0.1 imagePullPolicy:IfNotPresent mainApplicationFile:local:///opt/spark/examples/jars/spark-examples_2.13-4.0.1.jar mainClass:org.apache.spark.examples.SparkPi mode:cluster restartPolicy:map[type:Never] sparkConf:map[spark.blockManager.port:8081 spark.driver.blockManager.port:8082 spark.driver.port:8080 spark.port.maxRetries:0] sparkVersion:4.0.1 type:Scala volumes:[map[emptyDir:map[] name:spark-work-dir]]] status:map[applicationState:map[state:SUBMITTED] driverInfo:map[podName:spark-pi-dakfqvm5v4ss7nr1bqt0-driver webUIAddress:172.30.14.155:4040 webUIPort:4040 webUIServiceName:spark-pi-dakfqvm5v4ss7nr1bqt0-ui-svc] executionAttempts:1 lastSubmissionAttemptTime:2026-09-15T08:10:38Z sparkApplicationId:spark-b26fb4db07d24c738332d571a661392c submissionAttempts:1 submissionID:fd7d7b21-6e9f-41a0-a47f-80770a2b83aa terminationTime:<nil>]]
        to match expression
            <string>: .status.applicationState.state == "COMPLETED"
    sparkoperator_test.go:78: Cleaning up SparkApplication spark-pi-dakfqvm5v4ss7nr1bqt0
    testing.go:1913: test executed panic(nil) or runtime.Goexit: subtest may have called FailNow on a parent test]]></failure>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2" classname="" time="0.000">
			<system-out><![CDATA[    controller_test.go:767: Skipping tests for components/kueue
    controller_test.go:767: Skipping tests for components/modelregistry]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_3" classname="" time="0.000">
			<system-out><![CDATA[    controller_test.go:767: Skipping tests for components/mlflowoperator
    controller_test.go:767: Skipping tests for components/trustyai]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_4" classname="" time="0.000">
			<system-out><![CDATA[    controller_test.go:767: Skipping tests for components/modelsasservice]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_5" classname="" time="0.000">
			<system-out><![CDATA[    controller_test.go:767: Skipping tests for components/kserve
    controller_test.go:767: Skipping tests for components/kueue
    controller_test.go:767: Skipping tests for components/trainer]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_6" classname="" time="0.000">
			<system-out><![CDATA[    controller_test.go:767: Skipping tests for components/kserve-modelcache]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/services" classname="" time="1.520"></testcase>
		<testcase name="TestOdhOperator/services/group_1" classname="" time="0.000"></testcase>
		<testcase name="TestOdhOperator/services/group_1/auth" classname="" time="1.520"></testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway" classname="" time="1.360">
			<system-out><![CDATA[2026-09-15T08:15:41Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.5.1"}, "Cluster": {"type":"OpenShift","version":"4.20.12"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_GatewayConfig_creation" classname="" time="0.030">
			<system-out><![CDATA[    helper_test.go:157: Validating GatewayConfig resource
2026-09-15T08:15:41Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.5.1"}, "Cluster": {"type":"OpenShift","version":"4.20.12"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/auth/Validate_Auth_system_initialization" classname="" time="0.190">
			<system-out><![CDATA[    helper_test.go:157: GatewayConfig validation completed]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_Gateway_infrastructure" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_OAuth_client_and_secret_creation" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_authentication_proxy_deployment" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_unauthenticated_access_redirects_to_login" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_OIDC_proxy_secret_creation" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_OIDC_authentication_proxy_deployment" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_OIDC_token_forwarding_to_dashboard" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_OIDC_unauthenticated_access_redirects_to_login" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_kube-auth-proxy_TLS_args_match_cluster_APIServer" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_HorizontalPodAutoscaler_creation" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_NetworkPolicy_creation" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_OAuth_callback_HTTPRoute" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_EnvoyFilter_creation" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_EDS_endpoint_discovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_Gateway_ready_status" classname="" time="0.020">
			<system-out><![CDATA[    helper_test.go:157: Validating Gateway ready status
    helper_test.go:157: Gateway ready status validation completed]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_dashboard_redirect_resources" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/auth/Validate_addition_of_RoleBinding_when_group_is_added" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/auth/Validate_removal_of_bindings_when_a_group_is_removed" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/auth/Validate_CEL_blocks_Auth_update_with_invalid_groups" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/auth/Validate_CEL_allows_Auth_with_valid_groups" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/Operator_Resilience_E2E_Tests" classname="" time="0.210">
			<system-out><![CDATA[2026-09-15T08:15:42Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.5.1"}, "Cluster": {"type":"OpenShift","version":"4.20.12"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/Operator_Resilience_E2E_Tests/Validate_operator_deployment_health" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/Operator_Resilience_E2E_Tests/Validate_leader_election_behavior" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/Operator_Resilience_E2E_Tests/Validate_components_deployment_success" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/Operator_Resilience_E2E_Tests/Validate_components_deployment_failure" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/Operator_Resilience_E2E_Tests/Validate_missing_CRD_handling" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/Operator_Resilience_E2E_Tests/Validate_RBAC_restriction_handling" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/V2_to_V3_upgrade_E2E_Tests" classname="" time="0.210">
			<skipped message="Skipped"><![CDATA[2026-09-15T08:15:42Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.5.1"}, "Cluster": {"type":"OpenShift","version":"4.20.12"}}
    controller_test.go:767: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/Deletion_ConfigMap_E2E_Tests" classname="" time="0.270">
			<system-out><![CDATA[2026-09-15T08:15:42Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.5.1"}, "Cluster": {"type":"OpenShift","version":"4.20.12"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/Deletion_ConfigMap_E2E_Tests/Validate_creation_of_configmap_with_deletion_disabled" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier3]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/Deletion_ConfigMap_E2E_Tests/Validate_that_owned_namespaces_are_not_deleted" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier3]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/upgrade_DSC_and_DSCI_v1_API" classname="" time="0.210">
			<skipped message="Skipped"><![CDATA[2026-09-15T08:15:42Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.5.1"}, "Cluster": {"type":"OpenShift","version":"4.20.12"}}
    controller_test.go:767: Skipping test: passed tag: Smoke, test tags: [Tier3]]]></skipped>
		</testcase>
		<testcase name="TestLogPodsSectionContinuesWithPartialData" classname="" time="0.000"></testcase>
		<testcase name="TestResolveEnabledTests" classname="" time="0.000"></testcase>
		<testcase name="TestResolveEnabledTests/all_enabled_when_no_flags" classname="" time="0.000"></testcase>
		<testcase name="TestResolveEnabledTests/explicit_inclusion_limits_to_named_tests" classname="" time="0.000"></testcase>
		<testcase name="TestResolveEnabledTests/exclusion_removes_from_full_set" classname="" time="0.000"></testcase>
		<testcase name="TestResolveEnabledTests/inclusion_and_exclusion_combined" classname="" time="0.000"></testcase>
		<testcase name="TestRunSingle" classname="" time="0.000"></testcase>
		<testcase name="TestRunSingle/runs_only_the_named_test" classname="" time="0.000"></testcase>
		<testcase name="TestRunSingle/runs_only_the_named_test/monitoring" classname="" time="0.000"></testcase>
		<testcase name="TestRunSingle/skips_when_group_is_disabled" classname="" time="0.000"></testcase>
		<testcase name="TestRunSingle/skips_when_group_is_disabled/inner" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    testgroup_test.go:78: Test group services is disabled]]></skipped>
		</testcase>
		<testcase name="TestRunSingle/skips_when_test_not_found" classname="" time="0.000"></testcase>
		<testcase name="TestRunSingle/skips_when_test_not_found/inner" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    testgroup_test.go:85: Test nonexistent is disabled by flags in group services]]></skipped>
		</testcase>
		<testcase name="TestRunSingle/respects_flags_disabling_the_test" classname="" time="0.000"></testcase>
		<testcase name="TestRunSingle/respects_flags_disabling_the_test/inner" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    testgroup_test.go:94: Test monitoring is disabled by flags in group services]]></skipped>
		</testcase>
		<testcase name="TestRunExcluding" classname="" time="0.000"></testcase>
		<testcase name="TestRunExcluding/runs_all_except_excluded" classname="" time="0.000"></testcase>
		<testcase name="TestRunExcluding/runs_all_except_excluded/services" classname="" time="0.000"></testcase>
		<testcase name="TestRunExcluding/runs_all_except_excluded/services/group_1" classname="" time="0.000"></testcase>
		<testcase name="TestRunExcluding/runs_all_except_excluded/services/group_1/auth" classname="" time="0.000"></testcase>
		<testcase name="TestRunExcluding/runs_all_except_excluded/services/group_1/gateway" classname="" time="0.000"></testcase>
		<testcase name="TestRunExcluding/skips_when_group_is_disabled" classname="" time="0.000"></testcase>
		<testcase name="TestRunExcluding/skips_when_group_is_disabled/inner" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    testgroup_test.go:128: Test group services is disabled]]></skipped>
		</testcase>
		<testcase name="TestRunExcluding/skips_entirely_when_excluding_only_remaining_test" classname="" time="0.000"></testcase>
		<testcase name="TestRunExcluding/skips_entirely_when_excluding_only_remaining_test/inner" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    testgroup_test.go:142: No tests remaining in group single after excluding only]]></skipped>
		</testcase>
		<testcase name="TestIsTestEnabled" classname="" time="0.000"></testcase>
		<testcase name="TestIsTestEnabled/enabled_by_default" classname="" time="0.000"></testcase>
		<testcase name="TestIsTestEnabled/disabled_by_exclusion_flag" classname="" time="0.000"></testcase>
		<system-out><![CDATA[    controller_test.go:767: Skipping tests for components/trainer
    controller_test.go:767: Skipping tests for components/trainingoperator
    controller_test.go:767: Skipping tests for components/workbenches
2026/09/15 08:15:38 TEST FAILURE DETECTED: group 1
2026/09/15 08:15:38 === RUNNING CLUSTER DIAGNOSTICS (triggered by test:group 1) ===
2026/09/15 08:15:39 === CLUSTER STATE ===
2026/09/15 08:15:39 Failed to collect node data: unhealthy nodes: ip-10-0-1-126.us-east-2.compute.internal (Ready=False)
2026/09/15 08:15:39 === DEPLOYMENTS ===
2026/09/15 08:15:39 Namespace: redhat-ods-applications
2026/09/15 08:15:39   Deployment agent-ops-ui: 1/1 (READY)
2026/09/15 08:15:39   Deployment ai-gateway-operator: 1/1 (READY)
2026/09/15 08:15:39   Deployment automl-ui: 1/1 (READY)
2026/09/15 08:15:39   Deployment autorag-ui: 1/1 (READY)
2026/09/15 08:15:39   Deployment dashboard-operator: 1/1 (READY)
2026/09/15 08:15:39   Deployment dashboard-redirect: 2/2 (READY)
2026/09/15 08:15:39   Deployment data-science-pipelines-operator-controller-manager: 1/1 (READY)
2026/09/15 08:15:39   Deployment eval-hub-ui: 1/1 (READY)
2026/09/15 08:15:39   Deployment feast-operator-controller-manager: 1/1 (READY)
2026/09/15 08:15:39   Deployment gen-ai-ui: 1/1 (READY)
2026/09/15 08:15:39   Deployment kserve-controller-manager: 1/1 (READY)
2026/09/15 08:15:39   Deployment kserve-module-controller-manager: 1/1 (READY)
2026/09/15 08:15:39   Deployment kubeflow-trainer-controller-manager: 1/1 (READY)
2026/09/15 08:15:39   Deployment kuberay-operator: 1/1 (READY)
2026/09/15 08:15:39   Deployment llmisvc-controller-manager: 1/1 (READY)
2026/09/15 08:15:39   Deployment maas-controller: 1/1 (READY)
2026/09/15 08:15:39   Deployment maas-ui: 1/1 (READY)
2026/09/15 08:15:39   Deployment mlflow-operator-controller-manager: 1/1 (READY)
2026/09/15 08:15:39   Deployment mlflow-ui: 1/1 (READY)
2026/09/15 08:15:39   Deployment model-registry-operator-controller-manager: 1/1 (READY)
2026/09/15 08:15:39   Deployment model-registry-ui: 1/1 (READY)
2026/09/15 08:15:39   Deployment model-serving-api: 1/1 (READY)
2026/09/15 08:15:39   Deployment notebook-controller-deployment: 1/1 (READY)
2026/09/15 08:15:39   Deployment odh-model-controller: 1/1 (READY)
2026/09/15 08:15:39   Deployment odh-notebook-controller-manager: 1/1 (READY)
2026/09/15 08:15:39   Deployment ogx-k8s-operator-controller-manager: 1/1 (READY)
2026/09/15 08:15:39   Deployment opendatahub-feast-operator: 1/1 (READY)
2026/09/15 08:15:39   Deployment opendatahub-ogx-operator: 1/1 (READY)
2026/09/15 08:15:39   Deployment rhods-dashboard: 2/2 (READY)
2026/09/15 08:15:39   Deployment spark-operator-controller: 2/2 (READY)
2026/09/15 08:15:39   Deployment spark-operator-webhook: 2/2 (READY)
2026/09/15 08:15:39   Deployment trustyai-service-operator-controller-manager: 1/1 (READY)
2026/09/15 08:15:39   Deployment workbenches-operator: 1/1 (READY)
2026/09/15 08:15:39 Namespace: kube-system
2026/09/15 08:15:39 === PODS ===
2026/09/15 08:15:39 Failed to collect pod data: kube-system/global-pull-secret-syncer-cxzdd: phase=Pending; kube-system/konnectivity-agent-2w5rg: container konnectivity-agent not ready; redhat-ods-applications/ai-gateway-operator-78d898b99c-c7j8b: container manager restarts=1; redhat-ods-applications/dashboard-operator-77cd98cd96-mdr2s: container manager restarts=1; redhat-ods-applications/feast-operator-controller-manager-8458d8fbf4-md7fm: container manager restarts=1; redhat-ods-applications/kserve-controller-manager-77dfbd957d-9pd5m: container manager restarts=1; redhat-ods-applications/kserve-module-controller-manager-75c55478bb-6tvkw: container manager restarts=1; redhat-ods-applications/kubeflow-trainer-controller-manager-76b67466cc-hfqzj: container manager restarts=1; redhat-ods-applications/kuberay-operator-7b67b8456f-g6l2f: container kuberay-operator restarts=1; redhat-ods-applications/llmisvc-controller-manager-6b69d9b6c9-lxx7r: container manager restarts=1; redhat-ods-applications/maas-controller-7
789bf564f-tkcrg: container manager restarts=1; redhat-ods-applications/model-registry-operator-controller-manager-67dc8f7654-fdxlc: container manager restarts=1; redhat-ods-applications/ogx-k8s-operator-controller-manager-bd4677579-7gdwj: container manager restarts=1; redhat-ods-applications/opendatahub-feast-operator-79c7c8448-jmg7c: container manager restarts=1; redhat-ods-applications/opendatahub-ogx-operator-6b6657bf88-zxk68: container manager restarts=1; redhat-ods-applications/spark-operator-controller-688f5486c8-q79mt: container controller restarts=1; redhat-ods-applications/spark-operator-webhook-77866f5469-fltk4: container webhook restarts=1; redhat-ods-applications/trustyai-service-operator-controller-manager-5cbdf96ff6-tmqp6: container manager restarts=1; redhat-ods-applications/workbenches-operator-69694774df-csg8z: container manager restarts=1
2026/09/15 08:15:39 Continuing with partial pod data to collect logs from problematic containers
2026/09/15 08:15:39 Namespace: redhat-ods-applications
2026/09/15 08:15:39   Pod ai-gateway-operator-78d898b99c-c7j8b: Running
2026/09/15 08:15:39     Container manager: ready=true, restarts=1
2026/09/15 08:15:39     Container copy-manifests: ready=true, terminated: Completed (exit 0)
2026/09/15 08:15:39   Pod dashboard-operator-77cd98cd96-mdr2s: Running
2026/09/15 08:15:39     Container manager: ready=true, restarts=1
2026/09/15 08:15:39   Pod feast-operator-controller-manager-8458d8fbf4-md7fm: Running
2026/09/15 08:15:39     Container manager: ready=true, restarts=1
2026/09/15 08:15:39   Pod kserve-controller-manager-77dfbd957d-9pd5m: Running
2026/09/15 08:15:39     Container manager: ready=true, restarts=1
2026/09/15 08:15:39   Pod kserve-module-controller-manager-75c55478bb-6tvkw: Running
2026/09/15 08:15:39     Container manager: ready=true, restarts=1
2026/09/15 08:15:39   Pod kubeflow-trainer-controller-manager-76b67466cc-hfqzj: Running
2026/09/15 08:15:39     Container manager: ready=true, restarts=1
2026/09/15 08:15:39   Pod kuberay-operator-7b67b8456f-g6l2f: Running
2026/09/15 08:15:39     Container kuberay-operator: ready=true, restarts=1
2026/09/15 08:15:39   Pod llmisvc-controller-manager-6b69d9b6c9-lxx7r: Running
2026/09/15 08:15:39     Container manager: ready=true, restarts=1
2026/09/15 08:15:39   Pod maas-controller-7789bf564f-tkcrg: Running
2026/09/15 08:15:39     Container manager: ready=true, restarts=1
2026/09/15 08:15:39   Pod model-registry-operator-controller-manager-67dc8f7654-fdxlc: Running
2026/09/15 08:15:39     Container manager: ready=true, restarts=1
2026/09/15 08:15:39   Pod ogx-k8s-operator-controller-manager-bd4677579-7gdwj: Running
2026/09/15 08:15:39     Container manager: ready=true, restarts=1
2026/09/15 08:15:39   Pod opendatahub-feast-operator-79c7c8448-jmg7c: Running
2026/09/15 08:15:39     Container manager: ready=true, restarts=1
2026/09/15 08:15:39     Container copy-manifests: ready=true, terminated: Completed (exit 0)
2026/09/15 08:15:39   Pod opendatahub-ogx-operator-6b6657bf88-zxk68: Running
2026/09/15 08:15:39     Container manager: ready=true, restarts=1
2026/09/15 08:15:39     Container copy-manifests: ready=true, terminated: Completed (exit 0)
2026/09/15 08:15:39   Pod spark-operator-controller-688f5486c8-q79mt: Running
2026/09/15 08:15:39     Container controller: ready=true, restarts=1
2026/09/15 08:15:39   Pod spark-operator-webhook-77866f5469-fltk4: Running
2026/09/15 08:15:39     Container webhook: ready=true, restarts=1
2026/09/15 08:15:39   Pod trustyai-service-operator-controller-manager-5cbdf96ff6-tmqp6: Running
2026/09/15 08:15:39     Container manager: ready=true, restarts=1
2026/09/15 08:15:39   Pod workbenches-operator-69694774df-csg8z: Running
2026/09/15 08:15:39     Container manager: ready=true, restarts=1
2026/09/15 08:15:39 Namespace: kube-system
2026/09/15 08:15:39   Pod global-pull-secret-syncer-cxzdd: Pending
2026/09/15 08:15:39     Container global-pull-secret-syncer: ready=false, waiting: ContainerCreating
2026/09/15 08:15:39   Pod konnectivity-agent-2w5rg: Running
2026/09/15 08:15:39     Container konnectivity-agent: ready=false
2026/09/15 08:15:39 === OPERATOR STATUS ===
2026/09/15 08:15:39 Operator deployment: 3/3 ready
2026/09/15 08:15:39 === DSCI STATUS ===
2026/09/15 08:15:39 Failed to collect DSCI data: default-dsci unhealthy conditions: MonitoringStackAvailable=False; TempoAvailable=False; OpenTelemetryCollectorAvailable=False; AlertingAvailable=False; ThanosQuerierAvailable=False; PersesAvailable=False; NodeMetricsEndpointAvailable=False
2026/09/15 08:15:39 === DSC STATUS ===
2026/09/15 08:15:39 DSC default-dsc:
2026/09/15 08:15:39   Ready: True - 
2026/09/15 08:15:39   ProvisioningSucceeded: True - 
2026/09/15 08:15:39   ComponentsReady: True - 
2026/09/15 08:15:39   ModulesReady: True - 
2026/09/15 08:15:39   AIGatewayReady: True - 
2026/09/15 08:15:39   AIPipelinesReady: True - 
2026/09/15 08:15:39   BatchGatewayReady: False - Submodule ManagementState is set to Removed
2026/09/15 08:15:39   DashboardReady: True - 
2026/09/15 08:15:39   FeastOperatorReady: True - 
2026/09/15 08:15:39   KserveLLMInferenceServiceDependencies: True - 
2026/09/15 08:15:39   KserveLLMInferenceServiceWideEPDependencies: True - 
2026/09/15 08:15:39   KserveReady: True - 
2026/09/15 08:15:39   KueueReady: True - 
2026/09/15 08:15:39   MCPLifecycleOperatorReady: False - Module ManagementState is set to Removed
2026/09/15 08:15:39   MLflowOperatorReady: True - MLflowOperator is ready to manage MLflow custom resources
2026/09/15 08:15:39   ModelRegistryReady: True - 
2026/09/15 08:15:39   ModelsAsAServiceReady: True - modelsAsAService is Managed and deployments are available
2026/09/15 08:15:39   OGXReady: True - 
2026/09/15 08:15:39   ProvisioningProgress: True - 
2026/09/15 08:15:39   RayReady: True - 
2026/09/15 08:15:39   SparkOperatorReady: True - 
2026/09/15 08:15:39   TrainerReady: True - 
2026/09/15 08:15:39   TrainingOperatorReady: False - Component ManagementState is set to Removed
2026/09/15 08:15:39   TrustyAIReady: True - 
2026/09/15 08:15:39   WorkbenchesReady: True - Workbenches component is ready. Warning: 18 ImageStream tag(s) failed to import: runtime-rocm-tensorflow:rocm-tensorflow (dockerimage.image.openshift.io "registry.redhat.io/rhoai/odh-pipeline-runtime-tensorflow-rocm-py312-...); tensorflow:3.5 (dockerimage.image.openshift.io "registry.redhat.io/rhoai/odh-workbench-jupyter-tensorflow-cuda-py312...); code-server-notebook:3.5 (dockerimage.image.openshift.io "registry.redhat.io/rhoai/odh-workbench-codeserver-datascience-cpu-py...); jupyter-rocm-minimal:3.5 (dockerimage.image.openshift.io "registry.redhat.io/rhoai/odh-workbench-jupyter-minimal-rocm-py312-rh...); runtime-tensorflow:tensorflow (dockerimage.image.openshift.io "registry.redhat.io/rhoai/odh-pipeline-runtime-tensorflow-cuda-py312-...); jupyter-pytorch-llmcompressor:3.5 (dockerimage.image.openshift.io "registry.redhat.io/rhoai/odh-workbench-jupyter-pytorch-llmcompressor...); jupyter-rocm-pytorch:3.5 (dockerimage.image.openshift.io "registry.redhat.io/rhoai/od
h-workbench-jupyter-pytorch-rocm-py312-rh...); runtime-minimal:minimal (dockerimage.image.openshift.io "registry.redhat.io/rhoai/odh-pipeline-runtime-minimal-cpu-py312-rhel...); runtime-pytorch-llmcompressor:pytorch-llmcompressor (dockerimage.image.openshift.io "registry.redhat.io/rhoai/odh-pipeline-runtime-pytorch-llmcompressor-...); runtime-rocm-pytorch:rocm-pytorch (dockerimage.image.openshift.io "registry.redhat.io/rhoai/odh-pipeline-runtime-pytorch-rocm-py312-rhe...); ... and 8 more
2026/09/15 08:15:39 === RECENT EVENTS ===
2026/09/15 08:15:39   08:15:37 WARN Pod/global-pull-secret-syncer-cxzdd: NetworkNotReady - network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?
2026/09/15 08:15:39   08:15:37 INFO Pod/konnectivity-agent-2w5rg: Created - Created container: konnectivity-agent
2026/09/15 08:15:39   08:15:37 INFO Pod/konnectivity-agent-2w5rg: Started - Started container konnectivity-agent
2026/09/15 08:15:39   08:15:35 WARN Pod/global-pull-secret-syncer-cxzdd: FailedMount - MountVolume.SetUp failed for volume "original-pull-secret" : object "kube-system"/"original-pull-secret" not registered
2026/09/15 08:15:39   08:15:35 WARN Pod/global-pull-secret-syncer-cxzdd: FailedMount - MountVolume.SetUp failed for volume "global-pull-secret" : object "kube-system"/"global-pull-secret" not registered
2026/09/15 08:15:39   08:15:32 INFO Pod/konnectivity-agent-2w5rg: Pulled - Successfully pulled image "quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:98880a999bdb21bf52b5b635d08fc59ccf26475da454e2306428e5da55db416b" in 12.351s (12.351s including waiting). Image size: 468939173 bytes.
2026/09/15 08:15:39   08:15:20 WARN Pod/spark-pi-dakfqvm5v4ss7nr1bqt0-driver: FailedScheduling - 0/3 nodes are available: 1 node(s) had untolerated taint {node.kubernetes.io/not-ready: }, 2 Insufficient cpu. preemption: 0/3 nodes are available: 1 Preemption is not helpful for scheduling, 2 No preemption victims found for incoming pod.
2026/09/15 08:15:39   08:15:20 INFO Pod/kube-apiserver-proxy-ip-10-0-1-126.us-east-2.compute.internal: Pulled - Successfully pulled image "quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:469457eef00af3ae395c1f8e4dcc354faa680360ba813a14bf2d260cf46887b4" in 2.001s (2.001s including waiting). Image size: 482987592 bytes.
2026/09/15 08:15:39   08:15:20 INFO Pod/kube-apiserver-proxy-ip-10-0-1-126.us-east-2.compute.internal: Created - Created container: haproxy
2026/09/15 08:15:39   08:15:20 INFO Pod/konnectivity-agent-2w5rg: Pulling - Pulling image "quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:98880a999bdb21bf52b5b635d08fc59ccf26475da454e2306428e5da55db416b"
2026/09/15 08:15:39   08:15:20 INFO Pod/kube-apiserver-proxy-ip-10-0-1-126.us-east-2.compute.internal: Started - Started container haproxy
2026/09/15 08:15:39   08:15:18 INFO Pod/global-pull-secret-syncer-cxzdd: Scheduled - Successfully assigned kube-system/global-pull-secret-syncer-cxzdd to ip-10-0-1-126.us-east-2.compute.internal
2026/09/15 08:15:39   08:15:18 INFO Pod/konnectivity-agent-2w5rg: Scheduled - Successfully assigned kube-system/konnectivity-agent-2w5rg to ip-10-0-1-126.us-east-2.compute.internal
2026/09/15 08:15:39   08:15:18 INFO DaemonSet/konnectivity-agent: SuccessfulCreate - Created pod: konnectivity-agent-2w5rg
2026/09/15 08:15:39   08:15:18 INFO DaemonSet/global-pull-secret-syncer: SuccessfulCreate - Created pod: global-pull-secret-syncer-cxzdd
2026/09/15 08:15:39   08:15:18 INFO Pod/kube-apiserver-proxy-ip-10-0-1-126.us-east-2.compute.internal: Pulling - Pulling image "quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:469457eef00af3ae395c1f8e4dcc354faa680360ba813a14bf2d260cf46887b4"
2026/09/15 08:15:39   08:15:17 WARN Pod/spark-pi-dakfqvm5v4ss7nr1bqt0-driver: FailedScheduling - 0/3 nodes are available: 1 node(s) had untolerated taint {node.kubernetes.io/not-ready: }, 2 Insufficient cpu. preemption: 0/3 nodes are available: 1 Preemption is not helpful for scheduling, 2 No preemption victims found for incoming pod.
2026/09/15 08:15:39   08:12:07 WARN Pod/konnectivity-agent-lx265: Unhealthy - Readiness probe failed: HTTP probe failed with statuscode: 503
2026/09/15 08:15:39   08:11:50 INFO Lease/odh-model-controller.opendatahub.io: LeaderElection - odh-model-controller-c89d8c8db-v76n4_aef708b5-2852-4242-8fb1-9b2305c9b6b1 became leader
2026/09/15 08:15:39   08:11:33 INFO Pod/odh-model-controller-c89d8c8db-v76n4: Started - Started container manager
2026/09/15 08:15:39   08:11:33 INFO Pod/odh-model-controller-c89d8c8db-v76n4: Created - Created container: manager
2026/09/15 08:15:39   08:11:32 INFO Pod/odh-model-controller-c89d8c8db-v76n4: Pulled - Successfully pulled image "quay.io/rhoai/odh-model-controller-rhel9@sha256:f0f81921f0d2cbe4d5933786f2e94a483cb4705ed5768c2cc819b675ca001a8f" in 3.23s (3.23s including waiting). Image size: 238865862 bytes.
2026/09/15 08:15:39   08:11:29 INFO Pod/odh-model-controller-c89d8c8db-v76n4: AddedInterface - Add eth0 [10.130.20.109/23] from ovn-kubernetes
2026/09/15 08:15:39   08:11:29 INFO Pod/odh-model-controller-c89d8c8db-v76n4: Pulling - Pulling image "quay.io/rhoai/odh-model-controller-rhel9@sha256:f0f81921f0d2cbe4d5933786f2e94a483cb4705ed5768c2cc819b675ca001a8f"
2026/09/15 08:15:39   08:11:28 INFO Pod/odh-model-controller-c89d8c8db-v76n4: Scheduled - Successfully assigned redhat-ods-applications/odh-model-controller-c89d8c8db-v76n4 to ip-10-0-1-61.us-east-2.compute.internal
2026/09/15 08:15:39   08:11:28 INFO ReplicaSet/odh-model-controller-c89d8c8db: SuccessfulCreate - Created pod: odh-model-controller-c89d8c8db-v76n4
2026/09/15 08:15:39   08:11:28 INFO Deployment/odh-model-controller: ScalingReplicaSet - Scaled up replica set odh-model-controller-c89d8c8db from 0 to 1
2026/09/15 08:15:39   08:10:54 WARN Pod/spark-pi-dakfqvm5v4ss7nr1bqt0-driver: FailedScheduling - 0/2 nodes are available: 2 Insufficient cpu. preemption: 0/2 nodes are available: 2 No preemption victims found for incoming pod.
2026/09/15 08:15:39   08:10:54 WARN Pod/spark-pi-dakfqvm5v4ss7nr1bqt0-driver: FailedScheduling - 0/2 nodes are available: 2 Insufficient cpu. preemption: 0/2 nodes are available: 2 No preemption victims found for incoming pod.
2026/09/15 08:15:39   08:10:48 INFO Lease/llminferenceservice-kserve-controller-manager: LeaderElection - llmisvc-controller-manager-6b69d9b6c9-lxx7r_474f22ff-55cc-4ae4-9018-b79c5c6a4acd became leader
2026/09/15 08:15:39   08:10:48 INFO Lease/kserve-module-controller-manager: LeaderElection - kserve-module-controller-manager-75c55478bb-6tvkw_953a4089-2b86-4370-9ffc-3e7bf0353342 became leader
2026/09/15 08:15:39   08:10:44 INFO Lease/dashboard.components.platform.opendatahub.io: LeaderElection - dashboard-operator-77cd98cd96-mdr2s_9e0a2a53-7377-411f-bf68-5a85d6c07e61 became leader
2026/09/15 08:15:39   08:10:44 INFO Lease/kserve-controller-manager-leader-lock: LeaderElection - kserve-controller-manager-77dfbd957d-9pd5m_1e15575d-cd21-4d1c-8985-9495d2a110ec became leader
2026/09/15 08:15:39   08:10:44 INFO Lease/ray-operator-leader: LeaderElection - kuberay-operator-7b67b8456f-g6l2f_b029ba9f-49f3-4052-a965-813e2eeab446 became leader
2026/09/15 08:15:39   08:10:44 INFO Lease/85f368d1.opendatahub.io: LeaderElection - model-registry-operator-controller-manager-67dc8f7654-fdxlc_bb5250bd-31f2-4d51-a856-157ab99d4059 became leader
2026/09/15 08:15:39   08:10:43 INFO Pod/odh-model-controller-c89d8c8db-5wtmj: Killing - Stopping container manager
2026/09/15 08:15:39   08:10:42 INFO Lease/opendatahub-ogx-operator-lock: LeaderElection - opendatahub-ogx-operator-6b6657bf88-zxk68_0f6852a7-6494-4246-a425-1a6c171ed778 became leader
2026/09/15 08:15:39   08:10:42 INFO Lease/maas-controller.models-as-a-service.opendatahub.io: LeaderElection - maas-controller-7789bf564f-tkcrg_abd960a0-cb4f-4a41-b9a1-e20a5f1bee73 became leader
2026/09/15 08:15:39   08:10:42 INFO Lease/54e06e98.ogx.io: LeaderElection - ogx-k8s-operator-controller-manager-bd4677579-7gdwj_c09af334-f498-425a-bbd5-e7e5276e500b became leader
2026/09/15 08:15:39   08:10:41 INFO Lease/odh-model-controller.opendatahub.io: LeaderElection - odh-model-controller-c89d8c8db-5wtmj_764612f8-2d91-4261-9f6f-c29120443d26 became leader
2026/09/15 08:15:39   08:10:41 INFO Lease/trainer.kubeflow.org: LeaderElection - kubeflow-trainer-controller-manager-76b67466cc-hfqzj_dc972c9d-199c-4a4a-8fc5-48e2dd5bd432 became leader
2026/09/15 08:15:39   08:10:41 INFO Lease/66483dbd.feast.dev: LeaderElection - feast-operator-controller-manager-8458d8fbf4-md7fm_ffe638e1-8bdd-47b2-8efa-4565c430a09c became leader
2026/09/15 08:15:39   08:10:40 INFO Lease/spark-operator-webhook-lock: LeaderElection - spark-operator-webhook-77866f5469-fltk4_0d869371-b8cb-4c9f-aba8-bc3110a213c2 became leader
2026/09/15 08:15:39 === RESOURCE QUOTAS ===
2026/09/15 08:15:39 No resource quotas found in monitored namespaces
2026/09/15 08:15:39 Classification for group 1: infrastructure/network (code=1003, confidence=medium)
2026/09/15 08:15:39   Evidence: event Pod/global-pull-secret-syncer-cxzdd: NetworkNotReady - network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?
FAILURE_CLASSIFICATION: {"category":"infrastructure","subcategory":"network","error_code":1003,"evidence":["event Pod/global-pull-secret-syncer-cxzdd: NetworkNotReady - network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?"],"confidence":"medium"}
2026/09/15 08:15:39 Diagnostics complete for failed test: group 1
    controller_test.go:767: Stopping: group 1 test failed.
2026/09/15 08:15:39 Circuit breaker: failure classified as infrastructure/network (code=1003)
2026/09/15 08:15:39 Circuit breaker: infrastructure/unknown failure recorded (1/3)
2026/09/15 08:15:39 TEST FAILURE DETECTED: components
2026/09/15 08:15:39 === RUNNING CLUSTER DIAGNOSTICS (triggered by test:components) ===
2026/09/15 08:15:40 === CLUSTER STATE ===
2026/09/15 08:15:40 Failed to collect node data: unhealthy nodes: ip-10-0-1-126.us-east-2.compute.internal (Ready=False)
2026/09/15 08:15:40 === DEPLOYMENTS ===
2026/09/15 08:15:40 Namespace: redhat-ods-applications
2026/09/15 08:15:40   Deployment agent-ops-ui: 1/1 (READY)
2026/09/15 08:15:40   Deployment ai-gateway-operator: 1/1 (READY)
2026/09/15 08:15:40   Deployment automl-ui: 1/1 (READY)
2026/09/15 08:15:40   Deployment autorag-ui: 1/1 (READY)
2026/09/15 08:15:40   Deployment dashboard-operator: 1/1 (READY)
2026/09/15 08:15:40   Deployment dashboard-redirect: 2/2 (READY)
2026/09/15 08:15:40   Deployment data-science-pipelines-operator-controller-manager: 1/1 (READY)
2026/09/15 08:15:40   Deployment eval-hub-ui: 1/1 (READY)
2026/09/15 08:15:40   Deployment feast-operator-controller-manager: 1/1 (READY)
2026/09/15 08:15:40   Deployment gen-ai-ui: 1/1 (READY)
2026/09/15 08:15:40   Deployment kserve-controller-manager: 1/1 (READY)
2026/09/15 08:15:40   Deployment kserve-module-controller-manager: 1/1 (READY)
2026/09/15 08:15:40   Deployment kubeflow-trainer-controller-manager: 1/1 (READY)
2026/09/15 08:15:40   Deployment kuberay-operator: 1/1 (READY)
2026/09/15 08:15:40   Deployment llmisvc-controller-manager: 1/1 (READY)
2026/09/15 08:15:40   Deployment maas-controller: 1/1 (READY)
2026/09/15 08:15:40   Deployment maas-ui: 1/1 (READY)
2026/09/15 08:15:40   Deployment mlflow-operator-controller-manager: 1/1 (READY)
2026/09/15 08:15:40   Deployment mlflow-ui: 1/1 (READY)
2026/09/15 08:15:40   Deployment model-registry-operator-controller-manager: 1/1 (READY)
2026/09/15 08:15:40   Deployment model-registry-ui: 1/1 (READY)
2026/09/15 08:15:40   Deployment model-serving-api: 1/1 (READY)
2026/09/15 08:15:40   Deployment notebook-controller-deployment: 1/1 (READY)
2026/09/15 08:15:40   Deployment odh-model-controller: 1/1 (READY)
2026/09/15 08:15:40   Deployment odh-notebook-controller-manager: 1/1 (READY)
2026/09/15 08:15:40   Deployment ogx-k8s-operator-controller-manager: 1/1 (READY)
2026/09/15 08:15:40   Deployment opendatahub-feast-operator: 1/1 (READY)
2026/09/15 08:15:40   Deployment opendatahub-ogx-operator: 1/1 (READY)
2026/09/15 08:15:40   Deployment rhods-dashboard: 2/2 (READY)
2026/09/15 08:15:40   Deployment spark-operator-controller: 2/2 (READY)
2026/09/15 08:15:40   Deployment spark-operator-webhook: 2/2 (READY)
2026/09/15 08:15:40   Deployment trustyai-service-operator-controller-manager: 1/1 (READY)
2026/09/15 08:15:40   Deployment workbenches-operator: 1/1 (READY)
2026/09/15 08:15:40 Namespace: kube-system
2026/09/15 08:15:40 === PODS ===
2026/09/15 08:15:40 Failed to collect pod data: redhat-ods-applications/ai-gateway-operator-78d898b99c-c7j8b: container manager restarts=1; redhat-ods-applications/dashboard-operator-77cd98cd96-mdr2s: container manager restarts=1; redhat-ods-applications/feast-operator-controller-manager-8458d8fbf4-md7fm: container manager restarts=1; redhat-ods-applications/kserve-controller-manager-77dfbd957d-9pd5m: container manager restarts=1; redhat-ods-applications/kserve-module-controller-manager-75c55478bb-6tvkw: container manager restarts=1; redhat-ods-applications/kubeflow-trainer-controller-manager-76b67466cc-hfqzj: container manager restarts=1; redhat-ods-applications/kuberay-operator-7b67b8456f-g6l2f: container kuberay-operator restarts=1; redhat-ods-applications/llmisvc-controller-manager-6b69d9b6c9-lxx7r: container manager restarts=1; redhat-ods-applications/maas-controller-7789bf564f-tkcrg: container manager restarts=1; redhat-ods-applications/model-registry-operator-controller-manager-67dc8f7654-fdxlc: contai
ner manager restarts=1; redhat-ods-applications/ogx-k8s-operator-controller-manager-bd4677579-7gdwj: container manager restarts=1; redhat-ods-applications/opendatahub-feast-operator-79c7c8448-jmg7c: container manager restarts=1; redhat-ods-applications/opendatahub-ogx-operator-6b6657bf88-zxk68: container manager restarts=1; redhat-ods-applications/spark-operator-controller-688f5486c8-q79mt: container controller restarts=1; redhat-ods-applications/spark-operator-webhook-77866f5469-fltk4: container webhook restarts=1; redhat-ods-applications/trustyai-service-operator-controller-manager-5cbdf96ff6-tmqp6: container manager restarts=1; redhat-ods-applications/workbenches-operator-69694774df-csg8z: container manager restarts=1; kube-system/global-pull-secret-syncer-cxzdd: phase=Pending; kube-system/konnectivity-agent-2w5rg: container konnectivity-agent not ready
2026/09/15 08:15:40 Continuing with partial pod data to collect logs from problematic containers
2026/09/15 08:15:40 Namespace: redhat-ods-applications
2026/09/15 08:15:40   Pod ai-gateway-operator-78d898b99c-c7j8b: Running
2026/09/15 08:15:40     Container manager: ready=true, restarts=1
2026/09/15 08:15:40     Container copy-manifests: ready=true, terminated: Completed (exit 0)
2026/09/15 08:15:40   Pod dashboard-operator-77cd98cd96-mdr2s: Running
2026/09/15 08:15:40     Container manager: ready=true, restarts=1
2026/09/15 08:15:40   Pod feast-operator-controller-manager-8458d8fbf4-md7fm: Running
2026/09/15 08:15:40     Container manager: ready=true, restarts=1
2026/09/15 08:15:40   Pod kserve-controller-manager-77dfbd957d-9pd5m: Running
2026/09/15 08:15:40     Container manager: ready=true, restarts=1
2026/09/15 08:15:40   Pod kserve-module-controller-manager-75c55478bb-6tvkw: Running
2026/09/15 08:15:40     Container manager: ready=true, restarts=1
2026/09/15 08:15:40   Pod kubeflow-trainer-controller-manager-76b67466cc-hfqzj: Running
2026/09/15 08:15:40     Container manager: ready=true, restarts=1
2026/09/15 08:15:40   Pod kuberay-operator-7b67b8456f-g6l2f: Running
2026/09/15 08:15:40     Container kuberay-operator: ready=true, restarts=1
2026/09/15 08:15:40   Pod llmisvc-controller-manager-6b69d9b6c9-lxx7r: Running
2026/09/15 08:15:40     Container manager: ready=true, restarts=1
2026/09/15 08:15:40   Pod maas-controller-7789bf564f-tkcrg: Running
2026/09/15 08:15:40     Container manager: ready=true, restarts=1
2026/09/15 08:15:40   Pod model-registry-operator-controller-manager-67dc8f7654-fdxlc: Running
2026/09/15 08:15:40     Container manager: ready=true, restarts=1
2026/09/15 08:15:40   Pod ogx-k8s-operator-controller-manager-bd4677579-7gdwj: Running
2026/09/15 08:15:40     Container manager: ready=true, restarts=1
2026/09/15 08:15:40   Pod opendatahub-feast-operator-79c7c8448-jmg7c: Running
2026/09/15 08:15:40     Container manager: ready=true, restarts=1
2026/09/15 08:15:40     Container copy-manifests: ready=true, terminated: Completed (exit 0)
2026/09/15 08:15:40   Pod opendatahub-ogx-operator-6b6657bf88-zxk68: Running
2026/09/15 08:15:40     Container manager: ready=true, restarts=1
2026/09/15 08:15:40     Container copy-manifests: ready=true, terminated: Completed (exit 0)
2026/09/15 08:15:40   Pod spark-operator-controller-688f5486c8-q79mt: Running
2026/09/15 08:15:40     Container controller: ready=true, restarts=1
2026/09/15 08:15:40   Pod spark-operator-webhook-77866f5469-fltk4: Running
2026/09/15 08:15:40     Container webhook: ready=true, restarts=1
2026/09/15 08:15:40   Pod trustyai-service-operator-controller-manager-5cbdf96ff6-tmqp6: Running
2026/09/15 08:15:40     Container manager: ready=true, restarts=1
2026/09/15 08:15:40   Pod workbenches-operator-69694774df-csg8z: Running
2026/09/15 08:15:40     Container manager: ready=true, restarts=1
2026/09/15 08:15:40 Namespace: kube-system
2026/09/15 08:15:40   Pod global-pull-secret-syncer-cxzdd: Pending
2026/09/15 08:15:40     Container global-pull-secret-syncer: ready=false, waiting: ContainerCreating
2026/09/15 08:15:40   Pod konnectivity-agent-2w5rg: Running
2026/09/15 08:15:40     Container konnectivity-agent: ready=false
2026/09/15 08:15:40 === OPERATOR STATUS ===
2026/09/15 08:15:40 Operator deployment: 3/3 ready
2026/09/15 08:15:40 === DSCI STATUS ===
2026/09/15 08:15:40 Failed to collect DSCI data: default-dsci unhealthy conditions: MonitoringStackAvailable=False; TempoAvailable=False; OpenTelemetryCollectorAvailable=False; AlertingAvailable=False; ThanosQuerierAvailable=False; PersesAvailable=False; NodeMetricsEndpointAvailable=False
2026/09/15 08:15:40 === DSC STATUS ===
2026/09/15 08:15:40 DSC default-dsc:
2026/09/15 08:15:40   Ready: True - 
2026/09/15 08:15:40   ProvisioningSucceeded: True - 
2026/09/15 08:15:40   ComponentsReady: True - 
2026/09/15 08:15:40   ModulesReady: True - 
2026/09/15 08:15:40   AIGatewayReady: True - 
2026/09/15 08:15:40   AIPipelinesReady: True - 
2026/09/15 08:15:40   BatchGatewayReady: False - Submodule ManagementState is set to Removed
2026/09/15 08:15:40   DashboardReady: True - 
2026/09/15 08:15:40   FeastOperatorReady: True - 
2026/09/15 08:15:40   KserveLLMInferenceServiceDependencies: True - 
2026/09/15 08:15:40   KserveLLMInferenceServiceWideEPDependencies: True - 
2026/09/15 08:15:40   KserveReady: True - 
2026/09/15 08:15:40   KueueReady: True - 
2026/09/15 08:15:40   MCPLifecycleOperatorReady: False - Module ManagementState is set to Removed
2026/09/15 08:15:40   MLflowOperatorReady: True - MLflowOperator is ready to manage MLflow custom resources
2026/09/15 08:15:40   ModelRegistryReady: True - 
2026/09/15 08:15:40   ModelsAsAServiceReady: True - modelsAsAService is Managed and deployments are available
2026/09/15 08:15:40   OGXReady: True - 
2026/09/15 08:15:40   ProvisioningProgress: True - 
2026/09/15 08:15:40   RayReady: True - 
2026/09/15 08:15:40   SparkOperatorReady: True - 
2026/09/15 08:15:40   TrainerReady: True - 
2026/09/15 08:15:40   TrainingOperatorReady: False - Component ManagementState is set to Removed
2026/09/15 08:15:40   TrustyAIReady: True - 
2026/09/15 08:15:40   WorkbenchesReady: True - Workbenches component is ready. Warning: 18 ImageStream tag(s) failed to import: runtime-rocm-tensorflow:rocm-tensorflow (dockerimage.image.openshift.io "registry.redhat.io/rhoai/odh-pipeline-runtime-tensorflow-rocm-py312-...); tensorflow:3.5 (dockerimage.image.openshift.io "registry.redhat.io/rhoai/odh-workbench-jupyter-tensorflow-cuda-py312...); code-server-notebook:3.5 (dockerimage.image.openshift.io "registry.redhat.io/rhoai/odh-workbench-codeserver-datascience-cpu-py...); jupyter-rocm-minimal:3.5 (dockerimage.image.openshift.io "registry.redhat.io/rhoai/odh-workbench-jupyter-minimal-rocm-py312-rh...); runtime-tensorflow:tensorflow (dockerimage.image.openshift.io "registry.redhat.io/rhoai/odh-pipeline-runtime-tensorflow-cuda-py312-...); jupyter-pytorch-llmcompressor:3.5 (dockerimage.image.openshift.io "registry.redhat.io/rhoai/odh-workbench-jupyter-pytorch-llmcompressor...); jupyter-rocm-pytorch:3.5 (dockerimage.image.openshift.io "registry.redhat.io/rhoai/od
h-workbench-jupyter-pytorch-rocm-py312-rh...); runtime-minimal:minimal (dockerimage.image.openshift.io "registry.redhat.io/rhoai/odh-pipeline-runtime-minimal-cpu-py312-rhel...); runtime-pytorch-llmcompressor:pytorch-llmcompressor (dockerimage.image.openshift.io "registry.redhat.io/rhoai/odh-pipeline-runtime-pytorch-llmcompressor-...); runtime-rocm-pytorch:rocm-pytorch (dockerimage.image.openshift.io "registry.redhat.io/rhoai/odh-pipeline-runtime-pytorch-rocm-py312-rhe...); ... and 8 more
2026/09/15 08:15:40 === RECENT EVENTS ===
2026/09/15 08:15:40   08:15:39 WARN Pod/global-pull-secret-syncer-cxzdd: NetworkNotReady - network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?
2026/09/15 08:15:40   08:15:37 INFO Pod/konnectivity-agent-2w5rg: Created - Created container: konnectivity-agent
2026/09/15 08:15:40   08:15:37 INFO Pod/konnectivity-agent-2w5rg: Started - Started container konnectivity-agent
2026/09/15 08:15:40   08:15:35 WARN Pod/global-pull-secret-syncer-cxzdd: FailedMount - MountVolume.SetUp failed for volume "original-pull-secret" : object "kube-system"/"original-pull-secret" not registered
2026/09/15 08:15:40   08:15:35 WARN Pod/global-pull-secret-syncer-cxzdd: FailedMount - MountVolume.SetUp failed for volume "global-pull-secret" : object "kube-system"/"global-pull-secret" not registered
2026/09/15 08:15:40   08:15:32 INFO Pod/konnectivity-agent-2w5rg: Pulled - Successfully pulled image "quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:98880a999bdb21bf52b5b635d08fc59ccf26475da454e2306428e5da55db416b" in 12.351s (12.351s including waiting). Image size: 468939173 bytes.
2026/09/15 08:15:40   08:15:20 WARN Pod/spark-pi-dakfqvm5v4ss7nr1bqt0-driver: FailedScheduling - 0/3 nodes are available: 1 node(s) had untolerated taint {node.kubernetes.io/not-ready: }, 2 Insufficient cpu. preemption: 0/3 nodes are available: 1 Preemption is not helpful for scheduling, 2 No preemption victims found for incoming pod.
2026/09/15 08:15:40   08:15:20 INFO Pod/kube-apiserver-proxy-ip-10-0-1-126.us-east-2.compute.internal: Pulled - Successfully pulled image "quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:469457eef00af3ae395c1f8e4dcc354faa680360ba813a14bf2d260cf46887b4" in 2.001s (2.001s including waiting). Image size: 482987592 bytes.
2026/09/15 08:15:40   08:15:20 INFO Pod/kube-apiserver-proxy-ip-10-0-1-126.us-east-2.compute.internal: Created - Created container: haproxy
2026/09/15 08:15:40   08:15:20 INFO Pod/konnectivity-agent-2w5rg: Pulling - Pulling image "quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:98880a999bdb21bf52b5b635d08fc59ccf26475da454e2306428e5da55db416b"
2026/09/15 08:15:40   08:15:20 INFO Pod/kube-apiserver-proxy-ip-10-0-1-126.us-east-2.compute.internal: Started - Started container haproxy
2026/09/15 08:15:40   08:15:18 INFO Pod/global-pull-secret-syncer-cxzdd: Scheduled - Successfully assigned kube-system/global-pull-secret-syncer-cxzdd to ip-10-0-1-126.us-east-2.compute.internal
2026/09/15 08:15:40   08:15:18 INFO Pod/konnectivity-agent-2w5rg: Scheduled - Successfully assigned kube-system/konnectivity-agent-2w5rg to ip-10-0-1-126.us-east-2.compute.internal
2026/09/15 08:15:40   08:15:18 INFO DaemonSet/konnectivity-agent: SuccessfulCreate - Created pod: konnectivity-agent-2w5rg
2026/09/15 08:15:40   08:15:18 INFO DaemonSet/global-pull-secret-syncer: SuccessfulCreate - Created pod: global-pull-secret-syncer-cxzdd
2026/09/15 08:15:40   08:15:18 INFO Pod/kube-apiserver-proxy-ip-10-0-1-126.us-east-2.compute.internal: Pulling - Pulling image "quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:469457eef00af3ae395c1f8e4dcc354faa680360ba813a14bf2d260cf46887b4"
2026/09/15 08:15:40   08:15:17 WARN Pod/spark-pi-dakfqvm5v4ss7nr1bqt0-driver: FailedScheduling - 0/3 nodes are available: 1 node(s) had untolerated taint {node.kubernetes.io/not-ready: }, 2 Insufficient cpu. preemption: 0/3 nodes are available: 1 Preemption is not helpful for scheduling, 2 No preemption victims found for incoming pod.
2026/09/15 08:15:40   08:12:07 WARN Pod/konnectivity-agent-lx265: Unhealthy - Readiness probe failed: HTTP probe failed with statuscode: 503
2026/09/15 08:15:40   08:11:50 INFO Lease/odh-model-controller.opendatahub.io: LeaderElection - odh-model-controller-c89d8c8db-v76n4_aef708b5-2852-4242-8fb1-9b2305c9b6b1 became leader
2026/09/15 08:15:40   08:11:33 INFO Pod/odh-model-controller-c89d8c8db-v76n4: Started - Started container manager
2026/09/15 08:15:40   08:11:33 INFO Pod/odh-model-controller-c89d8c8db-v76n4: Created - Created container: manager
2026/09/15 08:15:40   08:11:32 INFO Pod/odh-model-controller-c89d8c8db-v76n4: Pulled - Successfully pulled image "quay.io/rhoai/odh-model-controller-rhel9@sha256:f0f81921f0d2cbe4d5933786f2e94a483cb4705ed5768c2cc819b675ca001a8f" in 3.23s (3.23s including waiting). Image size: 238865862 bytes.
2026/09/15 08:15:40   08:11:29 INFO Pod/odh-model-controller-c89d8c8db-v76n4: AddedInterface - Add eth0 [10.130.20.109/23] from ovn-kubernetes
2026/09/15 08:15:40   08:11:29 INFO Pod/odh-model-controller-c89d8c8db-v76n4: Pulling - Pulling image "quay.io/rhoai/odh-model-controller-rhel9@sha256:f0f81921f0d2cbe4d5933786f2e94a483cb4705ed5768c2cc819b675ca001a8f"
2026/09/15 08:15:40   08:11:28 INFO Pod/odh-model-controller-c89d8c8db-v76n4: Scheduled - Successfully assigned redhat-ods-applications/odh-model-controller-c89d8c8db-v76n4 to ip-10-0-1-61.us-east-2.compute.internal
2026/09/15 08:15:40   08:11:28 INFO ReplicaSet/odh-model-controller-c89d8c8db: SuccessfulCreate - Created pod: odh-model-controller-c89d8c8db-v76n4
2026/09/15 08:15:40   08:11:28 INFO Deployment/odh-model-controller: ScalingReplicaSet - Scaled up replica set odh-model-controller-c89d8c8db from 0 to 1
2026/09/15 08:15:40   08:10:54 WARN Pod/spark-pi-dakfqvm5v4ss7nr1bqt0-driver: FailedScheduling - 0/2 nodes are available: 2 Insufficient cpu. preemption: 0/2 nodes are available: 2 No preemption victims found for incoming pod.
2026/09/15 08:15:40   08:10:54 WARN Pod/spark-pi-dakfqvm5v4ss7nr1bqt0-driver: FailedScheduling - 0/2 nodes are available: 2 Insufficient cpu. preemption: 0/2 nodes are available: 2 No preemption victims found for incoming pod.
2026/09/15 08:15:40   08:10:48 INFO Lease/llminferenceservice-kserve-controller-manager: LeaderElection - llmisvc-controller-manager-6b69d9b6c9-lxx7r_474f22ff-55cc-4ae4-9018-b79c5c6a4acd became leader
2026/09/15 08:15:40   08:10:48 INFO Lease/kserve-module-controller-manager: LeaderElection - kserve-module-controller-manager-75c55478bb-6tvkw_953a4089-2b86-4370-9ffc-3e7bf0353342 became leader
2026/09/15 08:15:40   08:10:44 INFO Lease/dashboard.components.platform.opendatahub.io: LeaderElection - dashboard-operator-77cd98cd96-mdr2s_9e0a2a53-7377-411f-bf68-5a85d6c07e61 became leader
2026/09/15 08:15:40   08:10:44 INFO Lease/kserve-controller-manager-leader-lock: LeaderElection - kserve-controller-manager-77dfbd957d-9pd5m_1e15575d-cd21-4d1c-8985-9495d2a110ec became leader
2026/09/15 08:15:40   08:10:44 INFO Lease/ray-operator-leader: LeaderElection - kuberay-operator-7b67b8456f-g6l2f_b029ba9f-49f3-4052-a965-813e2eeab446 became leader
2026/09/15 08:15:40   08:10:44 INFO Lease/85f368d1.opendatahub.io: LeaderElection - model-registry-operator-controller-manager-67dc8f7654-fdxlc_bb5250bd-31f2-4d51-a856-157ab99d4059 became leader
2026/09/15 08:15:40   08:10:43 INFO Pod/odh-model-controller-c89d8c8db-5wtmj: Killing - Stopping container manager
2026/09/15 08:15:40   08:10:42 INFO Lease/opendatahub-ogx-operator-lock: LeaderElection - opendatahub-ogx-operator-6b6657bf88-zxk68_0f6852a7-6494-4246-a425-1a6c171ed778 became leader
2026/09/15 08:15:40   08:10:42 INFO Lease/maas-controller.models-as-a-service.opendatahub.io: LeaderElection - maas-controller-7789bf564f-tkcrg_abd960a0-cb4f-4a41-b9a1-e20a5f1bee73 became leader
2026/09/15 08:15:40   08:10:42 INFO Lease/54e06e98.ogx.io: LeaderElection - ogx-k8s-operator-controller-manager-bd4677579-7gdwj_c09af334-f498-425a-bbd5-e7e5276e500b became leader
2026/09/15 08:15:40   08:10:41 INFO Lease/odh-model-controller.opendatahub.io: LeaderElection - odh-model-controller-c89d8c8db-5wtmj_764612f8-2d91-4261-9f6f-c29120443d26 became leader
2026/09/15 08:15:40   08:10:41 INFO Lease/trainer.kubeflow.org: LeaderElection - kubeflow-trainer-controller-manager-76b67466cc-hfqzj_dc972c9d-199c-4a4a-8fc5-48e2dd5bd432 became leader
2026/09/15 08:15:40   08:10:41 INFO Lease/66483dbd.feast.dev: LeaderElection - feast-operator-controller-manager-8458d8fbf4-md7fm_ffe638e1-8bdd-47b2-8efa-4565c430a09c became leader
2026/09/15 08:15:40   08:10:40 INFO Lease/spark-operator-webhook-lock: LeaderElection - spark-operator-webhook-77866f5469-fltk4_0d869371-b8cb-4c9f-aba8-bc3110a213c2 became leader
2026/09/15 08:15:40 === RESOURCE QUOTAS ===
2026/09/15 08:15:40 No resource quotas found in monitored namespaces
2026/09/15 08:15:40 Classification for components: infrastructure/network (code=1003, confidence=medium)
2026/09/15 08:15:40   Evidence: event Pod/global-pull-secret-syncer-cxzdd: NetworkNotReady - network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?
FAILURE_CLASSIFICATION: {"category":"infrastructure","subcategory":"network","error_code":1003,"evidence":["event Pod/global-pull-secret-syncer-cxzdd: NetworkNotReady - network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?"],"confidence":"medium"}
2026/09/15 08:15:40 Diagnostics complete for failed test: components
    controller_test.go:495: Stopping: components test failed.
2026/09/15 08:15:40 Circuit breaker: failure classified as infrastructure/network (code=1003)
2026/09/15 08:15:40 Circuit breaker: infrastructure/unknown failure recorded (2/3)
    controller_test.go:767: Skipping tests for services/monitoring
Deletion Policy: Never. Skipping deletion tests.
2026/09/15 08:15:42 Circuit breaker: no trips during this run (threshold=3)
    controller_test.go:767: Skipping tests for services/monitoring]]></system-out>
	</testsuite>
</testsuites>
